p47035891
/
depends-public
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

optimize ruby parser

This commit is contained in:
Gang ZHANG 2019-01-06 15:12:52 +08:00
parent 9036160be1
commit b6482121d6
4 changed files with 47 additions and 21 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/main/antlr4/depends/extractor/ruby/RubyParser.g4
View File

@ -123,19 +123,19 @@ expr
| expr postfix=QUESTION
| prefix=(PLUS| MINUS|MUL|MOD|BIT_AND) expr
| expr LEFT_SBRACKET expr RIGHT_SBRACKET /* array access */
| prefix=DEFINED expr /* identifier definition test */
| expr bop=(DOT2|DOT3) expr? /* range */
| expr ','? MUL? ASSIGN crlfs? expr /* batch assign */
| prefix=DEFINED expr /* identifier definition test */
| expr bop=(DOT2|DOT3) expr? /* range */
| expr ','? MUL? ASSIGN crlfs? expr /* batch assign */
| expr assignOperator crlfs? expr /* assign */
| expr bop=PATTERN_MATCH expr /* pattern match */
| expr bop=BIT_NOT expr /* pattern match */
| expr bop=PATTERN_MATCH expr /* pattern match */
| expr bop=BIT_NOT expr /* pattern match */
| expr SIGH BIT_NOT expr /* pattern match */
| (not| BIT_NOT) expr /* logical not */
| expr (compareOperator) crlfs? expr /* compare logical */
| expr(logicalOperator) crlfs? expr /* logical join */
| expr(equalsOperator) crlfs? expr /* equal test */
| expr(mathOperator|bitOperator) crlfs? expr /* calcuation */
| expr QUESTION expr COLON expr /* cond?true_part:false_part */
| expr QUESTION expr COLON expr /* cond?true_part:false_part */
| expr block
| expr expr_statement_suffix
| expr dot_ref CLASS

1
src/main/java/depends/extractor/cpp/cdt/CdtCppEntitiesListener.java
View File

@ -26,7 +26,6 @@ import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTLinkageSpecification;
import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTProblemDeclaration;
import org.eclipse.cdt.internal.core.dom.parser.cpp.CPPASTVisibilityLabel;
import depends.entity.Expression;
import depends.entity.FunctionEntity;
import depends.entity.VarEntity;
import depends.entity.repo.EntityRepo;

9
src/main/java/depends/extractor/cpp/cdt/ExpressionUsage.java
View File

@ -1,16 +1,15 @@
package depends.extractor.cpp.cdt;
import org.eclipse.cdt.core.dom.ast.IASTNode;
import org.eclipse.cdt.core.dom.ast.IASTTypeIdExpression;
import org.eclipse.cdt.core.dom.ast.IASTUnaryExpression;
import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTNewExpression;
import org.eclipse.cdt.core.dom.ast.IASTBinaryExpression;
import org.eclipse.cdt.core.dom.ast.IASTCastExpression;
import org.eclipse.cdt.core.dom.ast.IASTDeclaration;
import org.eclipse.cdt.core.dom.ast.IASTDeclarator;
import org.eclipse.cdt.core.dom.ast.IASTExpression;
import org.eclipse.cdt.core.dom.ast.IASTFunctionCallExpression;
import org.eclipse.cdt.core.dom.ast.IASTIdExpression;
import org.eclipse.cdt.core.dom.ast.IASTLiteralExpression;
import org.eclipse.cdt.core.dom.ast.IASTNode;
import org.eclipse.cdt.core.dom.ast.IASTTypeIdExpression;
import org.eclipse.cdt.core.dom.ast.IASTUnaryExpression;
import org.eclipse.cdt.core.dom.ast.cpp.ICPPASTNewExpression;
import depends.entity.Expression;
import depends.entity.repo.IdGenerator;

46
src/main/java/depends/extractor/ruby/RubyFileParser.java
View File

@ -1,24 +1,28 @@
package depends.extractor.ruby;
import java.io.IOException;
import java.time.Duration;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import org.antlr.v4.runtime.BailErrorStrategy;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
import org.antlr.v4.runtime.DefaultErrorStrategy;
import org.antlr.v4.runtime.Lexer;
import org.antlr.v4.runtime.RuleContext;
import org.antlr.v4.runtime.atn.PredictionMode;
import org.antlr.v4.runtime.misc.ParseCancellationException;
import org.antlr.v4.runtime.tree.ParseTreeWalker;
import depends.entity.repo.EntityRepo;
import depends.extractor.FileParser;
public class RubyFileParser implements FileParser {
private static final long MAX_PARSE_TIME_PER_FILE = 30000L;
private String fileFullPath;
private EntityRepo entityRepo;
private ExecutorService executor;
@ -34,27 +38,51 @@ public class RubyFileParser implements FileParser {
CharStream input = CharStreams.fromFileName(fileFullPath);
Lexer lexer = new RubyLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
final Future<RubyParser> handler = executor.submit(new Callable<RubyParser>() {
final Future<RuleContext> handler = executor.submit(new Callable<RuleContext>() {
@Override
public RubyParser call() throws Exception {
return new RubyParser(tokens);
public RuleContext call() throws Exception {
RubyParser parser = new RubyParser(tokens);
return parser.prog();
}
/* The following function is try to optimize performance with SLL mode, but
* the actual result shows that there is no improvement
* so we remove it again */
@SuppressWarnings("unused")
public RuleContext callWithFallBack() throws Exception {
RubyParser parser = new RubyParser(tokens);
parser.setErrorHandler(new BailErrorStrategy());
parser.getInterpreter().setPredictionMode(PredictionMode.SLL);
try {
return parser.prog();
} catch (final ParseCancellationException e) {
// fall-back to LL mode parsing if SLL fails
tokens.reset();
parser.reset();
parser.removeErrorListeners();
parser.setErrorHandler(new DefaultErrorStrategy());
parser.getInterpreter().setPredictionMode(PredictionMode.LL);
return parser.prog();
}
}
});
RubyParser parser =null;
RuleContext rootContext =null;
try {
parser = handler.get(15000, TimeUnit.MILLISECONDS);
rootContext = handler.get(MAX_PARSE_TIME_PER_FILE, TimeUnit.MILLISECONDS);
} catch (TimeoutException | InterruptedException | ExecutionException e) {
System.err.println("time out of parse error in " + fileFullPath);
handler.cancel(true);
return;
}
if (parser==null) {
if (rootContext==null) {
System.err.println("parse error in " + fileFullPath);
return;
}
RubyListener bridge = new RubyListener(fileFullPath, entityRepo);
ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(bridge, parser.prog());
walker.walk(bridge, rootContext);
}
}