阅读背景:

pool.map的第二个参数想传入多个咋整?

来源:互联网 
from functools import  partial
from multiprocessing import Pool as ThreadPool
 pageurls=[]
        if maxpage:
            for page in range(2, maxpage + 1):
                pageurl = url + '&page=%s#mSearch' % page
                pageurls.append(pageurl)
            with ThreadPool() as pool:
                   results = pool.map(partial(get_one_page,level1=level1,level2=level2,level3=level3,name=name),pageurls)



def get_one_page(pageurl,level1,level2,level3,name):
    print("正在爬取的url", pageurl)
    p_list = parser_md_index(pageurl)
    for p in p_list:
        product_url, img, p_br, p_name, p_price, old_price = p['product_url'], p['img'], p['p_br'], p[
            'p_name'], p['p_price'], p['old_price']
        product_with(level1, level2, level3, name, p_br, p_name, p_price, old_price, product_url, img)
        name = re.sub('\"', '\'',  name)
        p_name = re.sub('\"', '\'', p_name)
        sql = 'INSERT INTO modern_product (level1,level2,level3,level4,p_br,p_name,p_price,old_price,product_url,img) VALUES ("{}","{}","{}","{}","{}","{}","{}","{}","{}","{}")'.format(
            level1, level2, level3, name, p_br, p_name, p_price, old_price, product_url, img)
        cur.execute(sql)
        conn.commit()from functools import  partial
from multipr



你的当前访问异常,请进行认证后继续阅读剩余内容。

分享到: