导入深交所年报代码 – 源码集中营

# -*- coding: utf-8 -*-
"""
Created on Sat Sep 14 15:34:11 2019
@author: gogogo
"""
import requests
import time
import pandas as pd
import random
import os
import json
# 定义爬取函数
def get_pdf_address(pageNum,start_date,end_date):
url = 'https://www.szse.cn/api/disc/announcement/annList?random=%s' % random.random()
headers = {'Accept': 'application/json, text/javascript, */*; q=0.01'
,'Accept-Encoding': 'gzip, deflate'
,'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
,'Content-Type': 'application/json'
,'Host': 'www.szse.cn'
,'Origin': 'https://www.szse.cn'
,'Proxy-Connection': 'keep-alive'
,'Referer': 'https://www.szse.cn/disclosure/listed/fixed/index.html'
,'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
,'X-Request-Type': 'ajax'
,'X-Requested-With': 'XMLHttpRequest'}
pagenum = int(pageNum)
payload = {"seDate":["{}-12-31".format(start_date),"{}-12-31".format(end_date)],"channelCode":["fixed_disc"],"bigCategoryId":["010301"],"pageSize":30,"pageNum":pagenum}
r = requests.post(url,headers = headers,data = json.dumps(payload))
result = r.json()
return result
#创建一个DataFrame储存爬取信息
data_download_pdf = pd.DataFrame(columns =['secCode','secName','url','title','publishTime'])
count = 0
url_head = 'https://disc.static.szse.cn/download/'
end = 2018
while end > 2017:
start = end  - 2
result = get_pdf_address(1,start,end)
print("开始爬取{}-12-31至{}-12-31的年报".format(start,end))
pages = int(result['announceCount']/30 + 1)
print("共%s页" % pages)
for i in range(1,pages):
print("爬取深交所年报下载地址第{}页".format(i))
print("完成{}%".format(round(i/pages,4) * 100))
result = get_pdf_address(i,start,end)
num = len(result['data'])
for each in range(num):
secCode = result['data'][each]['secCode'][0]
secName = result['data'][each]['secName'][0]
url = url_head + result['data'][each]['attachPath']
title = result['data'][each]['title']
publishTime = result['data'][each]['publishTime']
dict1 = {'secCode':secCode,'Name':secName,'url':url,'title':title,
'publishTime':publishTime}
data_download_pdf = data_download_pdf.append(dict1, ignore_index=True )
print('获取完成')
print("========================================")
time.sleep(random.uniform(1,2)) # 控制访问速度
end = end - 2
# 提取title中字符串获取年份
data_download_pdf['Year'] = data_download_pdf['title'].str.extract('([0-9]{4})')
file_path = "F:\深交所年报\"
headers={'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/76.0.3809.132 Safari/537.36'}
# 文件路径要以\结尾，如放入F盘年报文件夹，现在F盘创建文件夹，将路径写成   F:\年报\
for each in range(data_download_pdf.shape[0]):
# each = 1
# pdf_url = "https://disc.static.szse.cn/download//disc/disk02/finalpage/2019-07-05/dde0ce5e-e2c7-4c09-b6f4-a03ad9d593ee.PDF"
code = data_download_pdf.at[each,'secCode']
name = data_download_pdf.at[each,'secName'].replace("*","")
year = data_download_pdf.at[each,'Year']
print("开始下载{}，股票代码{}的{}年报".format(name,code,year))
file_name = "{}{}{}.pdf".format(code,name,year)
file_full_name = os.path.join(file_path, file_name)
file_full_name  = 'F:\1.pdf'
# print(file_full_name)
pdf_url = data_download_pdf.at[each,'url']
rs = requests.get(pdf_url,headers= headers, stream=True)
with open(file_full_name, "wb") as fp:
for chunk in rs.iter_content(chunk_size=10240):
if chunk:
fp.write(chunk)
time.sleep(random.uniform(1,2)) # 控制访问速度
print("===================下载完成==========================")

# -*- coding: utf-8 -*-
"""
Created on Sat Sep 14