p78659104
/
ossean
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

add loop to sync matchresult

This commit is contained in:
waterbeach 2017-03-06 23:08:25 +08:00
parent 13e29bb44f
commit facd45a674
2 changed files with 31 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
project_manager/src/main/java/com/ossean/projectmanager/projectsfilter/ProjectsFilter.java
View File

@ -48,6 +48,8 @@ public class ProjectsFilter {
} catch (InterruptedException e) {
logger.error(e);
}
startId = lastProjectDao.getMinId(0);
logger.info("restart Filter~");
}
else{
logger.info("Filtering projects......");

49
project_manager/src/main/java/com/ossean/projectmanager/syncmatchresult/SyncMatchResult.java
View File

@ -33,29 +33,38 @@ public class SyncMatchResult {
conf = new Configure("configure");
int startTableId = conf.getStartTableId();
int endTableId = conf.getEndTableId();
for (int i = startTableId; i <= endTableId; i++) {
String matchResultTableName = "relative_memo_to_open_source_projects_" + String.valueOf(i);
logger.info("starting sync " + matchResultTableName + "......");
List<RelativeMemoToOpenSourceProject> rmtospList = matchResultDao.getMatchResultsByHassync(matchResultTableName, 50000);
long startCircle = System.currentTimeMillis();
while(rmtospList !=null && rmtospList.size() > 0){
for (RelativeMemoToOpenSourceProject rmtosp : rmtospList) {
int curMemoId = rmtosp.getRelative_memo_id();
RelativeMemo curMemo = lastMemoDao.getById(curMemoId);
rmtosp.setCreated_time(curMemo.getCreated_time());
rmtosp.setMemo_type(curMemo.getMemo_type());
rmtosp.setReplies_num(curMemo.getReplies_num());
rmtosp.setView_num_crawled(curMemo.getView_num());
matchResultDao.syncMatchResult(matchResultTableName, rmtosp);
while(true){
logger.info("start match table~");
for (int i = startTableId; i <= endTableId; i++) {
String matchResultTableName = "relative_memo_to_open_source_projects_" + String.valueOf(i);
logger.info("starting sync " + matchResultTableName + "......");
List<RelativeMemoToOpenSourceProject> rmtospList = matchResultDao.getMatchResultsByHassync(matchResultTableName, 50000);
long startCircle = System.currentTimeMillis();
while(rmtospList !=null && rmtospList.size() > 0){
for (RelativeMemoToOpenSourceProject rmtosp : rmtospList) {
int curMemoId = rmtosp.getRelative_memo_id();
RelativeMemo curMemo = lastMemoDao.getById(curMemoId);
rmtosp.setCreated_time(curMemo.getCreated_time());
rmtosp.setMemo_type(curMemo.getMemo_type());
rmtosp.setReplies_num(curMemo.getReplies_num());
rmtosp.setView_num_crawled(curMemo.getView_num());
matchResultDao.syncMatchResult(matchResultTableName, rmtosp);
}
logger.info("updated " + rmtospList.get(rmtospList.size()-1).getId() + "......");
rmtospList = matchResultDao.getMatchResultsByHassync(matchResultTableName, 50000);
}
logger.info("updated " + rmtospList.get(rmtospList.size()-1).getId() + "......");
rmtospList = matchResultDao.getMatchResultsByHassync(matchResultTableName, 50000);
long endCircle = System.currentTimeMillis();
logger.info(matchResultTableName+" sync stop cost : " + (endCircle - startCircle)/6000 +" s");
}
logger.info("all match table have sync over!!!!");
logger.info("......sleeping 15 min......");
try {
Thread.sleep(1000*60*15);// 筛选完成，休息15分钟
} catch (InterruptedException e) {
logger.error(e);
}
long endCircle = System.currentTimeMillis();
logger.info(matchResultTableName+" sync stop cost : " + (endCircle - startCircle)/6000 +" s");
}
logger.info("all match table have sync over!!!!");
}