对虎牙直播进行爬取,并对信息进行处置剖析

08.16爬虫练手

一.代码

import requests
from  lxml.html import etree

#我们先选个lol专区
response = requests.get("https://www.huya.com/g/lol")
response.encoding =response.apparent_encoding
response_html = etree.HTML(response.text)
#以上是网页获得和解析


#相干信息的xpath
urls_xpath = "//*[@id="js-live-list"]/li/a[1]/@href"
user_name_xpath = "//*[@id="js-live-list"]/li/span/span[1]/i/text()"
popularity_xpath = "//*[@id="js-live-list"]/li/span/span[2]/i[2]/text()"
titles_xpath = "//*[@id="js-live-list"]/li/a[2]/text()"

#依据xpath进行获得
urls = response_html.xpath(urls_xpath)
user_name = response_html.xpath(user_name_xpath)
popularitys = response_html.xpath(popularity_xpath)
titles = response_html.xpath(titles_xpath)

#对爬取下来信息进行处置
dic = zip(urls,user_name,popularitys,titles)
new_list = []
for url,name,popularity,title in dic:
    #print(f"主播名称:{name}")
    if not popularity.endswith("万"):
        popularity = round(int(popularity)/10000,3)
        popularity = str(popularity)+"万"
    # print(f"主播人气:{popularity}")  #这里我们发明人气有些是有万结尾有些没有,所以我们对信息进行处置
    # print(f"直播间url:{url}")
    # print(f"直播间题目:{title}")
    # print("-"*200)
    new_dict = {"name":name,"popularity":popularity,"url":url,"title":title}
    new_list.append(new_dict)

    
#依照人气进行排序
new_list.sort(key=lambda a:float(a["popularity"][:-1]))

#由于上面是人气按从低到高进行排序了,我们进进行下反转后打印
for data in new_list[::-1]:
    print(f"主播名称:{data["name"]}")
    print(f"主播人气:{data["popularity"]}")
    print(f"直播间url:{data["url"]}")
    print(f"直播间题目:{data["title"]}")
    print("-"*200)import request