import scrapy from xxx.items import WorkItem class XXXSpider(scrapy.Spider): name = "xxx" allowed_domains = ["example.com"] start_urls = ["https://example.com/xx/xx"] def parse(self, response): year_list = response.xpath('//ul[@class="p-accordion"]/li') for year in year_list[:2]: release_dates_url_of_year = year.xpath('.//div[@class="genre -s"]/a/@href').extract() for date_url in release_dates_url_of_year: yield scrapy.Request ( url = date_url, callback =self.date_detail_parse ) def date_detail_parse(self, response): work_list = response.xpath('.//div[@class="swiper-slide c-low--6"]/div') for work in work_list: actress_name = work.xpath('.//a[@class="name c-main-font-hover"]/text()').extract_first() if actress_name is not None: item = WorkItem() item['actress_name'] = actress_name item['image_hover'] = work.xpath('.//img[@class="c-main-bg lazyload"]/@data-src').extract_first() work_detail_url = work.xpath('.//a[@class="img hover"]/@href').extract_first() if work_detail_url is not None: yield scrapy.Request ( url = work_detail_url, callback = self.work_detail_pares, meta = {'workItem' : item} ) def work_detail_pares(self, response): item = response.meta['workItem'] pics_list = response.xpath('.//div[@class="swiper-wrapper"]/div') pre_images = [] for pic in pics_list: img_url = pic.xpath('./img/@data-src').extract_first() pre_images.append(img_url) item['pre_images'] = pre_images item['name'] = response.xpath('.//div[@class="p-workPage l-wrap"]/h2/text()').extract_first().strip() item['id'] = response.xpath('.//span[@class="c-tag02 c-main-bg-hover c-main-bg"]/../text()').extract_first() item['company'] = 'xxx' item['release_date'] = response.xpath('.//div[@class="p-workPage__table"]/div[2]//div[@class="item"]/a/text()').extract_first() actress_detail_url = response.xpath('.//div[@class="p-workPage__table"]/div[1]//div[@class="item"]/a/@href').extract_first() yield scrapy.Request( url = actress_detail_url, callback = self.actress_detail_pase, meta = {'workItem' : item} ) def actress_detail_pase(self, response): item = response.meta['workItem'] item['actress_avatar'] = response.xpath('.//div[@class="swiper-slide"]/img/@data-src').extract_first() yield item
这是我的爬虫主程序,year_list我目前就只取了前 2 条数据做测试,发现爬取下来的数据跟网站上相比,很漏爬很多很多数据,而且每次爬取的数量都是同一个,网站上应该有至少几百条数据,而爬虫爬取下来,每次只能爬 35 条数据。而且每次爬取的数据内容都是变化的,这次没爬到的数据可能下一次再启动爬虫他就爬到了,但是总数永远是 35 。
你可以在 scrapy shell 一步一步手动操作,看看到底哪里不对。
可以看下日志是否有报错,你这个问题一般是哪里有错误,比如网站做了反爬,然后获取数据有问题。或者有可能是 url 有重复,被自动去冲了。
css 不会写-s 的吧?
先在每个 for 前面 print 一下 len 吧