This commit is contained in:
parent
7e4c268fb6
commit
2c47b105a0
|
@ -89,12 +89,16 @@ if __name__ == "__main__":
|
|||
.enableHiveSupport() \
|
||||
.getOrCreate()
|
||||
|
||||
count = 0
|
||||
while True:
|
||||
collect_crawl_info(spark)
|
||||
collect_news(spark)
|
||||
|
||||
get_last_day_count(spark)
|
||||
if count == 0 || count >100:
|
||||
get_last_day_count(spark)
|
||||
count = 1
|
||||
|
||||
time.sleep(10)
|
||||
count += 1
|
||||
|
||||
spark.stop()
|
Loading…
Reference in New Issue