65 lines
2.1 KiB
Ruby
65 lines
2.1 KiB
Ruby
|
namespace :solr_incremental_index do
|
||
|
|
task :index => :environment do
|
||
|
|
batch_size = 1000
|
||
|
|
unless SolrIncrementalIndexCursor.where(name: 'OpenSourceProject').exists?
|
||
|
|
osp_new_cursor=SolrIncrementalIndexCursor.new(name: 'OpenSourceProject', cursor: 0)
|
||
|
|
osp_new_cursor.save
|
||
|
|
end
|
||
|
|
unless SolrIncrementalIndexCursor.where(name: 'RelativeMemo').exists?
|
||
|
|
rm_new_cursor=SolrIncrementalIndexCursor.new(name: 'RelativeMemo', cursor: 0)
|
||
|
|
rm_new_cursor.save
|
||
|
|
end
|
||
|
|
|
||
|
|
cur_id=[]
|
||
|
|
osp_cursor=SolrIncrementalIndexCursor.where(name: 'OpenSourceProject').last
|
||
|
|
relative_memo_cursor=SolrIncrementalIndexCursor.where(name: 'RelativeMemo').last
|
||
|
|
cur_id[0]=osp_cursor.cursor
|
||
|
|
cur_id[1]=relative_memo_cursor.cursor
|
||
|
|
puts("OpenSourceProjectCursor: "+cur_id[0].to_s)
|
||
|
|
puts("RelativeMemo: "+cur_id[1].to_s)
|
||
|
|
|
||
|
|
while true
|
||
|
|
begin
|
||
|
|
osp_max_id = OpenSourceProject.maximum(:id)
|
||
|
|
rm_max_id = RelativeMemo.maximum(:id)
|
||
|
|
#Thread.new do
|
||
|
|
while osp_max_id>cur_id[0] || rm_max_id>cur_id[1] do
|
||
|
|
begin
|
||
|
|
if osp_max_id > cur_id[0]
|
||
|
|
OpenSourceProject.where("id > #{cur_id[0]} AND id < #{cur_id[0]+batch_size}").each do |osp_info|
|
||
|
|
#puts('start')
|
||
|
|
Sunspot.index(osp_info)
|
||
|
|
cur_id[0]=osp_info.id
|
||
|
|
puts("OpenSourceProject: "+cur_id[0].to_s)
|
||
|
|
#puts('inside')
|
||
|
|
end
|
||
|
|
end
|
||
|
|
osp_cursor.update_attribute('cursor', cur_id[0])
|
||
|
|
|
||
|
|
if rm_max_id > cur_id[1]
|
||
|
|
RelativeMemo.where("id > #{cur_id[1]} AND id < #{cur_id[1]+batch_size}").limit(1000).each do |memo|
|
||
|
|
Sunspot.index(memo)
|
||
|
|
cur_id[1]=memo.id
|
||
|
|
puts("RelativeMemo: "+cur_id[1].to_s)
|
||
|
|
end
|
||
|
|
end
|
||
|
|
relative_memo_cursor.update_attribute('cursor', cur_id[1])
|
||
|
|
#puts("while")
|
||
|
|
#sleep(600)
|
||
|
|
#puts('after sleep')
|
||
|
|
rescue Exception => e
|
||
|
|
puts e
|
||
|
|
end
|
||
|
|
end
|
||
|
|
|
||
|
|
rescue Exception => e
|
||
|
|
puts e
|
||
|
|
end
|
||
|
|
|
||
|
|
sleep(60)
|
||
|
|
|
||
|
|
# end
|
||
|
|
end
|
||
|
|
end
|
||
|
|
end
|