p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

11

This commit is contained in:
guange 2019-01-23 15:37:13 +08:00
parent ecf8c8ba75
commit b2d837ca12
1 changed files with 19 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
chapter1/crawler/taobao/utils.py
View File

@ -1,10 +1,11 @@
# coding=utf-8 # coding=utf-8
import time
import requests
import re
import time
import json import json
import re
import sys
import time
import pymysql import pymysql
import requests
def mysql_query(sql): def mysql_query(sql):
@ -17,13 +18,13 @@ def mysql_query(sql):
return data return data
def mysql_execute(sql): def mysql_execute(sql, port=3306):
print("execute: %s" % sql) print("execute: %s" % sql)
db = pymysql.connect(host="127.0.0.1", db = pymysql.connect(host="127.0.0.1",
user="root", user="root",
password="123456789", password="123456789",
database="sparkproject", database="sparkproject",
port=12346) port=port)
cursor = db.cursor() cursor = db.cursor()
try: try:
@ -59,7 +60,7 @@ def check_alive_proxy(ip, port):
raise RuntimeError("连接出错") raise RuntimeError("连接出错")
if __name__ == "__main__": def proxy(port):
r = requests.get( r = requests.get(
'https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list') 'https://raw.githubusercontent.com/fate0/proxylist/master/proxy.list')
print(r.status_code) print(r.status_code)
@ -73,7 +74,17 @@ if __name__ == "__main__":
print("%s:%d" % (o["host"], o["port"])) print("%s:%d" % (o["host"], o["port"]))
mysql_execute("insert into proxys (ip,port,checktime) values ('{}', {}, {})".format( mysql_execute("insert into proxys (ip,port,checktime) values ('{}', {}, {})".format(
o['host'], o['port'], check_time o['host'], o['port'], check_time
)) ), port)
except Exception as e: except Exception as e:
# print(e) # print(e)
pass pass
if __name__ == "__main__":
port = 3306
if len(sys.argv) > 1:
port = int(sys.argv[1])
while True:
proxy(port)
time.sleep(15*60)