From 5955efe77bc59bf1e2f7007d3e14e503a8454762 Mon Sep 17 00:00:00 2001
From: guange <8863824@gmail.com>
Date: Fri, 25 Jan 2019 15:54:57 +0800
Subject: [PATCH] =?UTF-8?q?=E6=B7=BB=E5=8A=A0=E4=BA=A7=E5=93=81=E8=A7=84?=
 =?UTF-8?q?=E6=A0=BC?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore                                    |   3 +-
 .../__pycache__/middlewares.cpython-37.pyc    | Bin 4326 -> 5147 bytes
 chapter1/crawler/taobao/items.py              |   9 ++++
 chapter1/crawler/taobao/middlewares.py        |  24 ++++++++-
 chapter1/crawler/taobao/pipelines.py          |  14 ++++-
 chapter1/crawler/taobao/spiders/jd.py         |  51 +++++++++++++++---
 chapter4/check_proxy.py                       |  31 +++++++++++
 7 files changed, 121 insertions(+), 11 deletions(-)
 create mode 100644 chapter4/check_proxy.py

diff --git a/.gitignore b/.gitignore
index ebc0014..40dad18 100644
--- a/.gitignore
+++ b/.gitignore
@@ -7,4 +7,5 @@ chapter1/crawler/datas/comments1/
 chapter1/crawler/datas/products/
 chapter1/crawler/taobao/settings.py
 chapter2/mysite/mysite/settings.py
-.env
\ No newline at end of file
+.env
+chapter4/results.csv
diff --git a/chapter1/crawler/taobao/__pycache__/middlewares.cpython-37.pyc b/chapter1/crawler/taobao/__pycache__/middlewares.cpython-37.pyc
index a0323b1ab08474ed549198bc4f6f30ecfff7493e..f5c641910037b1c3ed7bca4b9049eeaf188bc9df 100644
GIT binary patch
delta 2066
zcmZ`)&2QX96!-XJ?fvlXrp*TlAx)qO-L}foLRyptDiWeJDosNcm0*D^XUC)t-nE%=
z)Ff<_NXe}Slv=1#dr8!5B@P@pAtCV>aKHz|2@VJei9Z0|n@zH-l-Qb|-^{%CX5R0;
z8UH@?^g@2pG&2%BKcAodw5Mk;|I*fT=uwFhU$UBCViD1oEm?dOOM$QKt2NEi2<j+)
zs-|1IkX7HPrLDA(H9u3!T3I2d{9MhnOd;!jzE-dbM4FMPLDRbuO}oV%#p;0mn;Dvg
zfvgxf0|Pl~Lf;hqv(V4e0`v=_-$_YDDi!yh-!{!k=sW!Wg8SvF%lH2IiRkh7<O(Uo
zf02(!Upy&)(^-JXn1CWmX8?xcKjp8-&Vp3ZSSOBl0VFBr@>M@#-Pj>s)2L<H1H%{N
zZ<Wt3<5)>;vrM~dRX2>hpe~HtE#tS<SIArOJ@t_x76h~SZ?&_(xd>}l1+T0hU?LvW
zo(_n8q1H*2ER5ZXf6?fC2hL(Z4EM%fMnDG)Rocv2F&Hp`vGN>%q2+WVZDb6?$T6JR
z`?ouOmU?_64RT3gI(!(`iSP#lXv9o}Jc4lB@{POsFkaOc#|1x%;3v_U$W4>g6DCO^
zWI@&C+&)xVjCJGV31NOn@9Ve_?)U{_VJs=O?0Fc}xl7|G#^O6KIOsdgLC2AU;{?{?
za(cUb4h;qoh5$-(GuNkO<3apu`nvThs=tPC9$^?kWat7)C4`FzrzU3vW<P+&k#mup
z9Hxhjd>3T?kjJnf5<L)Q#*<uSdg0#e)U^HH!c65PuZC@VbuY5*?f9YjBe@pO=WkvW
z1ts{zI4qO7b?P#g+DidlKjDGlo*%|P<hQ4=YS}ehTdVp3ccBiBOB0Wb!=dU$6$rpX
zmY6*Bz_%?7=%aYIF!sp_dd5J{(x>j;y*)c!VK;CQ!P5*%EqBL0*(I3x3R;sLqRNJB
zWDT8Nii1q=)*#Hd65d&syez?8MX0<hSt?bi3SW(CaAl;Dt_7z~gW5a`*4FQMl=|)i
zhq+Y(*7zvURA^Js`1=5n)R0!l4x!`=!bT!_U!n2=*(8y=uQkX@>VPyP8`{<i+1C%G
zO_?eUnX0`Kpw=sWE_X|~?i|AvHm=@}qHw%iUh<Zy6FFB#UAkHgfT+>B8<lJ8P8gP*
z&?`exuC4RY6<~HFcW5)r$A@Vmdtsu60gFmTqWEq-N!wm%``#9hcw!u$BF=8YYKe;W
ziRK3m<%t@(Yf++vL3otE0ilEYXro(XFSYDO;Pwe92(l%TCmE8LGi0OxfB73--Vzp7
z<PRU@5M{yR8z2CWBqBR9yWEiWWgtq4l!mfHh}2MkN0pLZ__;t+Cg9fsl&Nz`ii^8F
zQ609-6HSm((vpnFy*iJadevnqaLp<J6^3P+s5Lioj&A1TQfITM)~YG-AHZUJ28|m4
z62xb~yD{7rU;JCX&zUQw*on%;BPtec7HrFJt5$D;%4298H)vq3rWuZvx*!E~iA{pc
z<`C}014Vz+<W=T`>vl6@5f+3m#>6e0q=Katw;G9Cn&CYmj)U;o7TECy5@Vtk&{f}k
ZR|j2=*B^!lubd-?ubfkg*Nf$1|9^L$rH=pr

delta 1317
zcmZ`&T~E_c818Ae_H=8zAs^u*0a3w?3yFjUgI_@$1Ta~`43Y-Yu$?AOJ33Fh1a5>x
zZ@e&yCtmR<{P+b#jb3@H=@0PAAK;boyk}!06Snl}`+eT?yl)RPzecS)rkRlVuauwO
z*y%a2`uFy^MkInf$!@%njkqp*a#gVv1UAKsRpWMC$T3f?YPKfixQDB{tqWQ864j)g
zMAD)}G=iT?1iQvF#WuOFlLXfjLbte{BnH<Fp=&9Eic;1*__@NS(I=#`Kj<p5*o1r^
z^|L+sZI8tpqykbPb#WMB8Rb>(9G9|j+6~qo4v|XTpzejV7Zg-!B(*d9xH-dqE1Sz;
z%f?P*>h4b2t%VhTTO@a6M*TJFm)#(6f?9>R)M-UeF&2AsO*GOd7g<5LMMH<WPwH+E
z2F+e*M@2fsJD6pg@xx)!K6Km}0&`kmeJYtrgBR{i;R<vR018kDIg!@Bs>U7N?7iBH
zH1<_}cO7#*t56l<bpXiZh9t-afC(T9w5!0m1@=K(pB02Dg0M(y-tAPTQpZd+gVxwx
z{9s(9H~AiigJ4|$<+CH)i%RSVUcdQY!oX<>+!l2mAK047W9I4mlf#fO0&t!~R&EUY
zES~FZC^2J?Lih!MivX7Z1oxLg$^c{m&ZK>edzbkWKm?8(8aA+w6LgFf(gPpAB)-cz
z7<Z)kV%LJmDCSq!mge)$-0EVngZ>CRPG!3iI?hw}&HRjV?4@;kO00|E5bk1|)ZOMA
z=@O^pliiLA6#9O_{#yI<u%a{vXUEH)AGmz>t;K`yW-H*UFD5SlzO=|}Zo5bzuh~&5
zx7R_@G>@~j`9fi3DPN?saERb)0i<@hR!us=9lQK#Foa?<mT?km!k$e>N{y809QQ}b
zpiH-FyH_5DVU5b%Vh;wc$L@8nZm12H*yc#}OOM=gNMYkC@GnZ8c6NksRDDwS+*=SE
S03#S-*+Am27)p93J^2r-!~4<z

diff --git a/chapter1/crawler/taobao/items.py b/chapter1/crawler/taobao/items.py
index aa8f4c5..5d4a5df 100644
--- a/chapter1/crawler/taobao/items.py
+++ b/chapter1/crawler/taobao/items.py
@@ -13,6 +13,15 @@ class ZhihuItem(scrapy.Item):
     # name = scrapy.Field()
     pass
 
+class ProductSpecItem(scrapy.Item):
+    cpu = scrapy.Field()
+    rom = scrapy.Field()
+    ram = scrapy.Field() 
+    resolution = scrapy.Field()
+    charge = scrapy.Field()
+    weight = scrapy.Field()
+    brand = scrapy.Field()
+
 class ProxyItem(scrapy.Item):
     ip = scrapy.Field()
     port = scrapy.Field()
diff --git a/chapter1/crawler/taobao/middlewares.py b/chapter1/crawler/taobao/middlewares.py
index fc129aa..0eae133 100644
--- a/chapter1/crawler/taobao/middlewares.py
+++ b/chapter1/crawler/taobao/middlewares.py
@@ -6,12 +6,14 @@
 # https://doc.scrapy.org/en/latest/topics/spider-middleware.html
 import pdb
 import time
+import json
+import requests
 
 from scrapy.http import HtmlResponse
 from selenium.common.exceptions import TimeoutException
 from selenium import webdriver
 from selenium.webdriver.support.wait import WebDriverWait
-
+from scrapy_splash import SplashRequest
 
 from scrapy import signals
 
@@ -144,3 +146,23 @@ class ZhihuChromeMiddleware(object):
     def spider_closed(self, spider, reason):
         print('驱动关闭')
         self.driver.close()
+
+
+class RandomProxyMiddleware(object):
+    _ip_lists = []
+
+    def get_ip(self):
+        if len(self._ip_lists) <= 0:
+            r = requests.get('http://bigdata1.educoder.net/myapp/api/proxys.json')
+            self._ip_lists = json.loads(r.text)
+
+        data = self._ip_lists.pop()
+        return "http://%s:%d" % (data['ip'], data['port'])
+            
+
+    def process_request(self, request, spider):
+        if isinstance(request, SplashRequest):
+            ip = self.get_ip()
+            print(ip)
+            request.meta['splash']['args']['proxy'] = ip
+            # request.meta['proxy'] = ip
\ No newline at end of file
diff --git a/chapter1/crawler/taobao/pipelines.py b/chapter1/crawler/taobao/pipelines.py
index 71ce326..2c49775 100644
--- a/chapter1/crawler/taobao/pipelines.py
+++ b/chapter1/crawler/taobao/pipelines.py
@@ -12,7 +12,7 @@ import happybase
 from scrapy.conf import settings
 from scrapy.pipelines.images import ImagesPipeline
 
-from taobao.items import JDProductItem, JDCommentItem, JDCommentSummary
+from taobao.items import JDProductItem, JDCommentItem, JDCommentSummary,ProductSpecItem
 from taobao.utils import check_alive_proxy
 from scrapy.exceptions import DropItem
 
@@ -129,5 +129,17 @@ class JDHbasePipeline(object):
                                     "comment:default_good_count": str(item["default_good_count"]),
                                     "comment:good_rate": str(item["good_rate"])
                                     })
+        elif isinstance(item, ProductSpecItem):
+            self.product_table.put(item["id"],
+                    {
+                        "spec:cpu": str(item["cpu"]),
+                        "spec:rom": str(item['rom']),
+                        "spec:ram": str(item["ram"]),
+                        "spec:resolution": str(item["resolution"]),
+                        "spec:charge": str(item["charge"]),
+                        "spec:weight": str(item["weight"]),
+                        "spec:brand": str(item["brand"]),
+                    }
+             )
 
         return item
diff --git a/chapter1/crawler/taobao/spiders/jd.py b/chapter1/crawler/taobao/spiders/jd.py
index 5ddef03..86f3a42 100644
--- a/chapter1/crawler/taobao/spiders/jd.py
+++ b/chapter1/crawler/taobao/spiders/jd.py
@@ -7,7 +7,7 @@ from scrapy import Request
 from scrapy.conf import settings
 from scrapy_splash import SplashRequest
 import pdb
-from taobao.items import JDProductItem, JDCommentItem, JDCommentSummary
+from taobao.items import JDProductItem, JDCommentItem, JDCommentSummary, ProductSpecItem
 import re
 
 from taobao.utils import now_time
@@ -21,11 +21,19 @@ class JdSpider(scrapy.Spider):
     ]
 
     custom_settings = {
-        'LOG_LEVEL': "WARN",
+        'LOG_LEVEL': "INFO",
         'ITEM_PIPELINES': {
             'taobao.pipelines.JDCleanDataPipeline': 300,
-            'taobao.pipelines.JDHbasePipeline': 400,
-        }
+            #'taobao.pipelines.JDHbasePipeline': 400,
+        },
+        'DOWNLOADER_MIDDLEWARES': {
+            'scrapy.downloadermiddlewares.retry.RetryMiddleware': 90,
+            #'taobao.middlewares.RandomProxyMiddleware': 100,
+            # 'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
+            'scrapy_splash.SplashCookiesMiddleware': 723,
+            'scrapy_splash.SplashMiddleware': 725,
+            'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 810,
+        },
     }
 
     def make_comment_url(self, product_id, page):
@@ -38,10 +46,34 @@ class JdSpider(scrapy.Spider):
         for url in self.start_urls:
             yield SplashRequest(url,
                                 self.parse,
+                                method='GET',
                                 args={})
 
+    def parse_spec(self, response):
+        #spec
+        pdb.set_trace()
+        cpu = response.xpath('//dt[text()="CPU型号"]/following-sibling::dd/text()').extract_first()
+        rom = response.xpath('//dt[text()="ROM"]/following-sibling::dd[2]/text()').extract_first()
+        ram = response.xpath('//dt[text()="RAM"]/following-sibling::dd[2]/text()').extract_first()
+        resolution = response.xpath('//dt[text()="分辨率"]/following-sibling::dd/text()').extract_first()
+        charge = response.xpath('//dt[text()="电池容量（mAh）"]/following-sibling::dd/text()').extract_first()
+        weight = response.xpath('//dt[text()="机身重量（g）"]/following-sibling::dd/text()').extract_first()
+        brand = response.xpath('//dt[text()="品牌"]/following-sibling::dd/text()').extract_first()
+
+        print(cpu, rom, ram, resolution, charge, weight, brand)
+
+        yield ProductSpecItem(
+            cpu = cpu,
+            rom = rom,
+            ram = ram,
+            resolution = resolution,
+            charge = charge,
+            weight = weight,
+            brand = brand, 
+        )
+
+
     def parse_comment(self, response):
-        # pdb.set_trace()
         product_id = response.meta["product_id"]
         page = response.meta["page"]
         print("抓取评论, %s, 当前 %d页" % (product_id, page))
@@ -128,9 +160,12 @@ class JdSpider(scrapy.Spider):
             if m:
                 product_id = m.group(1)
                 comment_url = self.make_comment_url(product_id, 0)
-                yield Request(comment_url, self.parse_comment,
-                              meta={'product_id': product_id, 'page': 0},
-                              priority=100)
+                # yield Request(comment_url, self.parse_comment,
+                #               meta={'product_id': product_id, 'page': 0},
+                #               priority=100)
+            
+            if url:
+                yield SplashRequest(url='https:'+url, callback=self.parse_spec)
 
         # 获取下一页
         next_page = response.xpath('//a[@class="pn-next"]/@href').extract_first()
diff --git a/chapter4/check_proxy.py b/chapter4/check_proxy.py
new file mode 100644
index 0000000..8eb6294
--- /dev/null
+++ b/chapter4/check_proxy.py
@@ -0,0 +1,31 @@
+import requests
+import time
+import re
+
+def check_alive_proxy(ip, port):
+    begin_time = int(time.time())
+    proxies = {
+        "http": "http://%s:%s" % (ip, port),
+        "https": "https://%s:%s" % (ip, port),
+    }
+    response = requests.get(
+        'http://2019.ip138.com/ic.asp', proxies=proxies, timeout=3)
+
+    check_time = int(time.time()) - begin_time
+
+    response.encoding = 'gb2312'
+    m = re.search(r'.+\[((\d+\.){3}\d+)\].+', response.text)
+    if m:
+        if m.group(1) == ip:
+            return check_time
+    raise RuntimeError("连接出错")
+
+if __name__ == "__main__":
+    with open('results.csv') as f:
+        for line in f:
+            try:
+                check_time = check_alive_proxy(line.strip(), 9999)
+                if check_time<5:
+                    print("%s:%d %d", line, 9999, check_time)
+            except Exception as e:
+                print(e)
\ No newline at end of file