p15679402
/
bigdata
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

.

This commit is contained in:
guange 2019-01-18 10:26:32 +08:00
parent ed358a8a59
commit 7b669f833b
2 changed files with 2 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
chapter2/mysite/myapp/views.py
View File

@ -13,7 +13,7 @@ def crawl(request):
# info = scrapy_client.get_scrapy_info() # info = scrapy_client.get_scrapy_info()
crawl_info = CrawlInfos.objects.order_by('-id').first() crawl_info = CrawlInfos.objects.order_by('-id').first()
platform_info = get_platform_info() platform_info = get_platform_info()
news = News.objects.order_by('-id')[0:20] news = News.objects.order_by('-id').all()[0:20]
return render(request, 'myapp/crawl.html', {"crawl_info": crawl_info, return render(request, 'myapp/crawl.html', {"crawl_info": crawl_info,
"platform_info":json.dumps(platform_info), "platform_info":json.dumps(platform_info),
"news": news}) "news": news})

1
chapter3/spark.py
View File

@ -52,6 +52,7 @@ def collect_crawl_info(spark):
def collect_news(spark): def collect_news(spark):
df = spark.sql("select * from jd_comment order by created_at desc limit 20") df = spark.sql("select * from jd_comment order by created_at desc limit 20")
mysql_execute("delete from news")
for row in df.rdd.collect(): for row in df.rdd.collect():
mysql_execute("insert into news (comment_time, content, comment_id) values ('{}', '{}', '{}')".format( mysql_execute("insert into news (comment_time, content, comment_id) values ('{}', '{}', '{}')".format(
row["comment_time"], row["content"], row["id"])) row["comment_time"], row["content"], row["id"]))