p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

.

This commit is contained in:
guange 2019-01-18 14:08:01 +08:00
parent cf8d1da3ff
commit 7e4c268fb6
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
chapter3/spark.py
View File

@ -67,10 +67,10 @@ def collect_news(spark):
def get_last_day_count(spark): def get_last_day_count(spark):
"""获取过去几天的采集量""" """获取过去几天的采集量"""
for i in range(5): for i in range(5):
df = spark.sql("select count(*) as N from jd where created_at like '"+getYesterday(i+1)+"%'") df = spark.sql("select count(*) as N from jd where created_at like '"+getYesterday(i)+"%'")
jd_last_count = df.rdd.collect()[0]["N"] jd_last_count = df.rdd.collect()[0]["N"]
df = spark.sql("select count(*) as N from jd_comment where created_at like '"+getYesterday(i+1)+"%'") df = spark.sql("select count(*) as N from jd_comment where created_at like '"+getYesterday(i)+"%'")
jd_comment_last_count = df.rdd.collect()[0]["N"] jd_comment_last_count = df.rdd.collect()[0]["N"]
mysql_execute("update last_day_counts set product_c = {}, comment_c = {} where last_day = {}".format( mysql_execute("update last_day_counts set product_c = {}, comment_c = {} where last_day = {}".format(