CQU毕设爬虫完善开发版

增加按专业文件夹分类

import requests
import time
import re
import os

class Spyder:
    def __init__(self, user_name, pass_word):
        spyder = requests.session()
        self.spyder = spyder
        self.user_name = user_name
        self.pass_word = pass_word
        self.title = None
        self.major = None



    def login(self):
        url = "https://bysj.cqu.edu.cn/bysj/index.jsp"

        data={
            "id": self.user_name,
            "pwd": self.pass_word,
            "type": "student",
            "btlogin": "登陆"

        }
        response = self.spyder.post(url,data=data)
        html = response.text

    def get_paper_title(self):
        url = "https://bysj.cqu.edu.cn/bysj/student/viewStudentPage.htm"
        response = self.spyder.get(url)
        response.encoding = "utf-8"
        html = response.text
        result = re.findall(".*>(.*)</td>.*", html)
        list_key = []
        list_value = []
        for i in range(1, len(result)):
            if i % 2 == 1:
                list_key.append(result[i].rstrip())
            else:
                list_value.append(result[i].rstrip())
        message = dict(zip(list_key, list_value))
        if '毕业设计题目：' in message:
            self.title = message["毕业设计题目："]
        if '专业:' in message:
            self.major = message["专业:"]

    def make_major_file(self):
        dirs = "D:/python学习/CQU毕设/"+self.major+"/"
        if not os.path.exists(dirs):
            os.makedirs(dirs)

    def get_paper(self):

        url = "https://bysj.cqu.edu.cn/bysj/thesiswork/student/manageThesisMaterials.htm"
        response = self.spyder.get(url)
        text = response.text
        url0 = "https://bysj.cqu.edu.cn"
        name = re.findall(r'<td class="title" colspan="7" height="30">(.*?)</td>',text)
        if(len(name)==0):
            print("获取失败")
            return 0
        else:
            stu_name=name[0][3:-2]
            print("正在获取"+stu_name)
        stu_name.strip()
        # 开题报告
        """
        report = re.findall(r'href="/go.html?url=(.*?)">查看开题报告</a></td>',text)
        if(len(report)==0):
            print("获取开题报告失败")
        else:
            print("开始下载开题报告")
            url = url0 + report[0]
            response = self.spyder.get(url)
            content = response.content
            with open(stu_name+"开题报告.doc","wb") as f:
                f.write(content)
        translate=re.findall(r'href="/go.html?url=(.*?)">查看译文</a></td>',text)
        if (len(translate) == 0):
            print("获取论文翻译失败")
        else:
            print("开始下载文献翻译")
            url = url0 + translate[0]
            response = self.spyder.get(url)
            content = response.content
            with open(stu_name + "文献翻译.doc", "wb") as f:
                f.write(content)
        """
        paper = re.findall(r'href="/go.html?url=(.*?)">查看论文</a></td>', text)
        self.make_major_file()
        if (len(paper) == 0):
            print("获取论文失败")
        else:
            print("开始下载论文")
            url = url0 + paper[0]
            response = self.spyder.get(url)
            content = response.content
            # 以下代码，用来解决文件名保存的问题
            self.title = str(self.title)
            file_name = stu_name + self.title+ ".doc"
            file_name = re.sub('\s', '', file_name)
            rstr = r"[\/\\:\*\?\"\<\>\|]"  # '/ \ : * ? " < > |'
            file_name = re.sub(rstr, "_", file_name)  # 替换为下划线
            file_name = "D:/python学习/CQU毕设/" + self.major + "/" + file_name
            with open(file_name, "wb") as f:
                f.write(content)


if __name__ == '__main__':
    for i in range(1000,6350):
        user_name = "2014" + str(i)
        pass_word = ""
        spyder = Spyder(user_name, pass_word)
        spyder.login()
        spyder.get_paper_title()
        spyder.get_paper()
        time.sleep(1)import requests
import time
import