用scrapy 大规模 无登陆 爬取 58同城 房产信息

快过年了，早上没什么事情就把之前自己写的一个小demo 拿出来记录在博客上

import scrapy

class CourseItem(scrapy.Item):
    title = scrapy.Field()   
    
    number = scrapy.Field()
    
    area = scrapy.Field()
    
    huxing = scrapy.Field()
    
    nature = scrapy.Field()

class MySpider(scrapy.Spider):
    
    name = "MySpiders"
    
    allowed_domains = ["xa.58.com"]
    
    start_urls = ["https://xa.58.com/chuzu/?PGTID=0d200001-001e-3800-de6e-943e5224b994&ClickID=1"]
    encoding='gb18030'
    def parse(self, response):
        item = CourseItem()
        for box  in response.xpath('//ul[@class="listUl"]/li'):
        
         item['title'] = box.xpath('.//div[@class="des"]/h2/a/text()').extract()
         try:
             item['title'] = item['title'][0].strip()
         except IndexError:
             item['title'] = ''    
         item['number'] = box.xpath('.//div[@class="listliright"]/div[@class="money"]/b/text()').extract()
         try:
             item['number'] = item['number'][0]
         except IndexError:
             item['number'] = ''
         item['area'] = box.xpath('.//div[@class="des"]/p[@class="add"]/a/text()').extract()
         try:
             item['area'] = item['area'][0]
         except IndexError:
             item['area'] = ''
         item['huxing'] = box.xpath('.//div[@class="des"]/p[@class="room"]/text()').extract()
         try:
             item['huxing'] = item['huxing'][0]
         except IndexError:
             item['huxing'] = ''
         item['nature'] = box.xpath('.//div[@class="des"]/div[@class="jjr"]/text()').extract()
         try:
             item['nature'] = item['nature'][0].strip()
         except IndexError:
             item['nature'] = ''
        
         yield item
         
         
         next_page = response.xpath('.//li[@id="bottom_ad_li"]/div[@class="pager"]/a[@class="next"]/@href').extract_first()
         
         
         
         if next_page:
            next_page = response.urljoin(str(next_page))
            
            yield scrapy.Request(next_page, callback=self.parse)
         else:
             print(None)
            import