Q:你知道html最常使用的标签有多少个吗
A:30个左右
from bs4 import BeautifulSoup
import urllib.request
import urllib.parse
def html_download(url):
if url is None:
return None
response = urllib.request.urlopen(url)
if 200 == response.status:
html = response.read()
return html
def html_parse(url, html):
soup = BeautifulSoup(html, "lxml")
tags_all = set()
tags_special = set()
tags_normal = {"html", "head", "body", "meta",
"title", "link", "script", "div",
"img", "span", "a", "p"} #基本上会用到的标签有12个
for tag in soup.find_all(True):
tags_all.add(tag.name)
if tag.name not in tags_normal:
tags_special.add(tag.name)
print("tags_all=%s" % tags_all)
print("url=%s, tags_all_length=%s" % (url, len(tags_all)))
tags_len.append(len(tags_all))
print("tags_special=%s" % tags_special)
print("url=%s, tags_special_length=%s" % (url, len(tags_special)))
print("---------------------------------------------------------------------\n")
if __name__ == "__main__":
root_urls = ["https://www.360.cn/",
"https://www.apple.com/cn/",
"https://www.taobao.com/",
"https://www.microsoftstore.com.cn/",
"https://www.amazon.cn",
"https://github.com/",
"https://en.wikipedia.org/wiki/Main_Page",
"https://developer.mozilla.org/zh-CN/"]
# 存放使用标签个数
tags_len = []
for url_item in root_urls:
html_content = html_download(url_item)
html_parse(url_item, html_content)
print(tags_len)
print(sum(tags_len)/len(tags_len))
print("crawler end")from bs4 i