p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

.

This commit is contained in:
guange 2019-01-17 21:23:22 +08:00
parent 26b2e9efab
commit 41e092becc
3 changed files with 30 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

0
chapter1/crawler/taobao/settings.py → chapter1/crawler/taobao/settings.py.template
View File

4
chapter1/crawler/taobao/spiders/jd.py
View File

@ -85,12 +85,12 @@ class JdSpider(scrapy.Spider):
# time.sleep(10) # time.sleep(10)
if page>settings['COMMENT_MAX_PAGE']: if page>settings['COMMENT_MAX_PAGE']:
print("评论抓取达到最大深度, %s, 页数: %d", product_id, page) print("评论抓取达到最大深度, %s, 页数: %d" % (product_id, page))
else: else:
yield Request(next_comment_url, self.parse_comment, yield Request(next_comment_url, self.parse_comment,
meta={'product_id': product_id, 'page': page}) meta={'product_id': product_id, 'page': page})
else: else:
print("评论抓取完成, %s, 页数: %d", product_id, page) print("评论抓取完成, %s, 页数: %d" % (product_id, page))
def parse(self, response): def parse(self, response):
items = response.xpath('//div[@id="plist"]//li[@class="gl-item"]') items = response.xpath('//div[@id="plist"]//li[@class="gl-item"]')