p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

.

This commit is contained in:
guange 2019-01-17 23:16:48 +08:00
parent 51c41dd0d1
commit b94235fb75
1 changed files with 16 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
chapter3/spark.py
View File

@ -42,6 +42,7 @@ if __name__ == "__main__":
.enableHiveSupport() \ .enableHiveSupport() \
.getOrCreate() .getOrCreate()
while True:
df = spark.sql("select count(*) as N from jd_comment") df = spark.sql("select count(*) as N from jd_comment")
jd_comment_count = df.rdd.collect()[0]["N"] jd_comment_count = df.rdd.collect()[0]["N"]
@ -60,4 +61,7 @@ if __name__ == "__main__":
mysql_execute("insert into crawl_infos (total_count, today_total_count, product_count, today_product_count, comment_count, today_comment_count) values ({},{},{},{},{},{})".format( mysql_execute("insert into crawl_infos (total_count, today_total_count, product_count, today_product_count, comment_count, today_comment_count) values ({},{},{},{},{},{})".format(
total_count, today_total_count, jd_count,jd_today_count, jd_comment_count, jd_comment_today_count) ) total_count, today_total_count, jd_count,jd_today_count, jd_comment_count, jd_comment_today_count) )
time.sleep(10)
spark.stop() spark.stop()