Python获取豆瓣《狼殿下》200条热门短评写入excel_lu_xuying的专栏

# 200条之后的评论需要登录才能查看，后续研究
# index需要修改

import urllib.request
import re
import pandas as pd

headers = {'User-Agent',
           'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/64.0.3282.186 Safari/537.36'}

df = pd.DataFrame({'comment_time': [0],
                   'user_name': [0],
                   'user_id': [0],
                   'star': [0],
                   'title': [0],
                   'comments': [0]})

for j in range(11):
    start_page = str(j * 20)
    url = "https://movie.douban.com/subject/26743567/comments?start=%s" %(start_page)  # 确定要爬取的入口链接 
    
    print(j, url)
    
    opener = urllib.request.build_opener()
    opener.addheaders = [headers]
    data = opener.open(url).read().decode('utf8')
    
    # 获取评论时间
    comment_time_pattern = re.compile('<span class="comment-time " title="(.*?)">', re.S)
    comment_time = re.findall(comment_time_pattern, data)

    # 获取用户名称
    user_name_pattern = re.compile('<a title="(.*?)" href="/go.html?url=https://www.douban.com/people/.*?/', re.S)
    user_name = re.findall(user_name_pattern, data)

    # 获取用户ID
    user_id_pattern = re.compile('<a title=".*?" href="/go.html?url=https://www.douban.com/people/(.*?)/', re.S)
    user_id = re.findall(user_id_pattern, data)

    # 获取用户星评（取值：10,20,30,40,50）
    star_pattern = re.compile('<span class="allstar(.*?) rating" title=".*?"></span>', re.S)
    star = re.findall(star_pattern, data)

    # 获取用户推荐指数（取值：很差，还行，推荐，力荐）
    title_pattern = re.compile('<span class="allstar.*? rating" title="(.*?)"></span>', re.S)
    title = re.findall(title_pattern, data)

    # 获取评论时间
    comments_pattern = re.compile('<span class="short">(.*?)</span>', re.S)
    comments = re.findall(comments_pattern, data)
    
    new_df = pd.DataFrame({'comment_time': comment_time,
                           'user_name': user_name,
                           'user_id': user_id,
                           'star': star,
                           'title': title,
                           'comments': comments})
    
    # print(j, new_df)
    
    df = df.append(new_df)

df.to_excel("The_Wolf_Douban_Comments.xls", index=True)
# 200条之后的评论需要登录才能查看，后续研究
# index需要修改

import u