p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

.

This commit is contained in:
guange 2019-01-18 14:09:33 +08:00
parent 7e4c268fb6
commit 2c47b105a0
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
chapter3/spark.py
View File

@ -89,12 +89,16 @@ if __name__ == "__main__":
.enableHiveSupport() \ .enableHiveSupport() \
.getOrCreate() .getOrCreate()
count = 0
while True: while True:
collect_crawl_info(spark) collect_crawl_info(spark)
collect_news(spark) collect_news(spark)
get_last_day_count(spark) if count == 0 || count >100:
get_last_day_count(spark)
count = 1
time.sleep(10) time.sleep(10)
count += 1
spark.stop() spark.stop()