This commit is contained in:
guange 2019-01-18 14:09:33 +08:00
parent 7e4c268fb6
commit 2c47b105a0
1 changed files with 5 additions and 1 deletions

View File

@ -89,12 +89,16 @@ if __name__ == "__main__":
.enableHiveSupport() \
.getOrCreate()
count = 0
while True:
collect_crawl_info(spark)
collect_news(spark)
get_last_day_count(spark)
if count == 0 || count >100:
get_last_day_count(spark)
count = 1
time.sleep(10)
count += 1
spark.stop()