p47035891
/
depends_public_new
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactory: refine responsbility of file parser/lang processor

This commit is contained in:
Gang ZHANG 2022-09-11 21:43:01 +08:00
parent b4f94214f3
commit 4286870bf9
20 changed files with 104 additions and 152 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
src/main/java/depends/entity/repo/EntityRepo.java
View File

@ -50,7 +50,9 @@ public interface EntityRepo extends IdGenerator {
Iterator<Entity> sortedFileIterator();
void addFile(FileEntity currentFileEntity);
void clear();
FileEntity getFileEntity(String fileFullPath);
void completeFile(String fileFullPath);
}

30
src/main/java/depends/entity/repo/InMemoryEntityRepo.java
View File

@ -4,6 +4,7 @@ import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.GenericName;
import depends.entity.MultiDeclareEntities;
import multilang.depends.util.file.FileUtil;
import java.util.*;
import java.util.Map.Entry;
@ -98,15 +99,34 @@ public class InMemoryEntityRepo extends SimpleIdGenerator implements EntityRepo
return allFileEntitiesByOrder.iterator();
}
@Override
public void addFile(FileEntity fileEntity) {
allFileEntitiesByOrder.add(fileEntity);
}
@Override
public void clear() {
allEntieisByName.clear();
allEntitiesById.clear();
allFileEntitiesByOrder.clear();
}
@Override
public FileEntity getFileEntity(String fileFullPath) {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
Entity entity = this.getEntity(fileFullPath);
if (entity ==null) return null;
if (entity instanceof FileEntity) return (FileEntity) entity;
if (entity instanceof MultiDeclareEntities){
MultiDeclareEntities multiDeclare = (MultiDeclareEntities) entity;
for (Entity theEntity: multiDeclare.getEntities()){
if (theEntity instanceof FileEntity){
return (FileEntity) theEntity;
}
}
}
return null;
}
@Override
public void completeFile(String fileFullPath) {
FileEntity fileEntity = getFileEntity(fileFullPath);
fileEntity.cacheAllExpressions();
allFileEntitiesByOrder.add(fileEntity);
}
}

22
src/main/java/depends/extractor/AbstractLangProcessor.java
View File

@ -160,32 +160,30 @@ abstract public class AbstractLangProcessor {
@Override
public void visit(File file) {
String fileFullPath = file.getAbsolutePath();
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
if (!fileFullPath.startsWith(inputSrcPath)) {
return;
}
if (isPhase2Files(fileFullPath)) {
} else {
parseFile(fileFullPath);
}
parseFile(fileFullPath, phase2Files);
}
});
fileTransversal.extensionFilter(this.fileSuffixes());
fileTransversal.travers(this.inputSrcPath);
for (String f : phase2Files) {
parseFile(f);
parseFile(f, phase2Files);
}
System.out.println("all files procceed successfully...");
}
protected void parseFile(String fileFullPath) {
protected void parseFile(String fileFullPath, Set<String> phase2Files) {
FileParser fileParser = createFileParser();
try {
System.out.println("parsing " + fileFullPath + "...");
fileParser.parse(FileUtil.uniqFilePath(fileFullPath));
if (fileParser.isPhase2Files(fileFullPath)){
phase2Files.add(fileFullPath);
}else {
fileParser.parse(fileFullPath);
}
} catch (IOException e) {
e.printStackTrace();
} catch (Exception e) {
@ -194,10 +192,6 @@ abstract public class AbstractLangProcessor {
}
}
protected boolean isPhase2Files(String fileFullPath) {
return false;
}
public List<String> includePaths() {
if (this.includePaths ==null) {
this.includePaths = buildIncludePath();

39
src/main/java/depends/extractor/FileParser.java
View File

@ -24,9 +24,44 @@ SOFTWARE.
package depends.extractor;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import multilang.depends.util.file.FileUtil;
import java.io.IOException;
public interface FileParser {
void parse(String filePath) throws IOException;
public abstract class FileParser {
protected EntityRepo entityRepo;
/**
* parse files
* @param filePath
* @throws IOException
*/
public final void parse(String filePath) throws IOException{
filePath = FileUtil.uniqFilePath(filePath);
/* If file already exist, skip it */
FileEntity fileEntity = entityRepo.getFileEntity(filePath);
if (fileEntity!=null) {
System.out.println("already parsed " + filePath + "...skip");
if (!fileEntity.isInProjectScope())
fileEntity.setInProjectScope(true);
}else {
System.out.println("parsing " + filePath + "...");
parseFile(filePath);
entityRepo.completeFile(filePath);
}
}
/**
* The actual file parser - it should put parsed entities into entityRepo;
* @param filePath - it is alread unique file path name
* @throws IOException
*/
protected abstract void parseFile(String filePath) throws IOException;
protected boolean isPhase2Files(String filePath){
return false;
}
}

4
src/main/java/depends/extractor/HandlerContext.java
View File

@ -57,10 +57,6 @@ public abstract class HandlerContext {
return currentFileEntity;
}
public void done() {
entityRepo.addFile(this.currentFile());
}
public TypeEntity foundNewType(GenericName name, Integer startLine) {
TypeEntity currentTypeEntity = new TypeEntity(name, this.latestValidContainer(),
idGenerator.generateId());

5
src/main/java/depends/extractor/cpp/CppFileParser.java
View File

@ -26,11 +26,8 @@ package depends.extractor.cpp;
import depends.entity.repo.EntityRepo;
public abstract class CppFileParser implements depends.extractor.FileParser {
protected EntityRepo entityRepo;
public abstract class CppFileParser extends depends.extractor.FileParser {
public CppFileParser(EntityRepo entityRepo) {
this.entityRepo = entityRepo;
}
}

9
src/main/java/depends/extractor/cpp/CppProcessor.java
View File

@ -97,13 +97,4 @@ public class CppProcessor extends AbstractLangProcessor {
depedencyTypes.add(IMPLLINK);
return depedencyTypes;
}
@Override
protected boolean isPhase2Files(String fileFullPath) {
if (fileFullPath.endsWith(".h") || fileFullPath.endsWith(".hh") || fileFullPath.endsWith(".hpp")
|| fileFullPath.endsWith(".hxx"))
return true;
return false;
}
}

35
src/main/java/depends/extractor/cpp/cdt/CdtCppFileParser.java
View File

@ -24,13 +24,10 @@ SOFTWARE.
package depends.extractor.cpp.cdt;
import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import depends.extractor.cpp.CppFileParser;
import depends.extractor.cpp.MacroRepo;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.eclipse.cdt.core.dom.ast.IASTTranslationUnit;
import java.io.IOException;
@ -49,10 +46,9 @@ public class CdtCppFileParser extends CppFileParser {
this.macroRepo= macroRepo;
}
@Override
public void parse(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
protected void parseFile(String fileFullPath) throws IOException {
Map<String, String> macroMap = new HashMap<>(macroRepo.getDefaultMap());
parse(FileUtil.uniqFilePath(fileFullPath),true,macroMap);
parse(fileFullPath,macroMap);
}
/**
@ -60,24 +56,13 @@ public class CdtCppFileParser extends CppFileParser {
* @param isInScope whether the parse is invoked by project file or an 'indirect' included file
* @return
*/
public void parse(String fileFullPath, boolean isInScope,Map<String, String> macroMap) {
/** If file already exist, skip it */
Entity fileEntity = entityRepo.getEntity(fileFullPath);
if (fileEntity!=null && fileEntity instanceof FileEntity) {
FileEntity t = ((FileEntity)fileEntity);
if (!t.isInProjectScope() && isInScope)
t.setInProjectScope(true);
return;
}
if (fileFullPath.contains("regex.h")){
System.out.println("stop");
}
public void parse(String fileFullPath,Map<String, String> macroMap) throws IOException {
CppVisitor bridge = new CppVisitor(fileFullPath, entityRepo, preprocessorHandler, bindingResolver);
IASTTranslationUnit tu = (new CDTParser(preprocessorHandler.getIncludePaths())).parse(fileFullPath,macroMap);
boolean containsIncludes = false;
for (String incl:preprocessorHandler.getDirectIncludedFiles(tu.getAllPreprocessorStatements(),fileFullPath)) {
CdtCppFileParser importedParser = new CdtCppFileParser(entityRepo, preprocessorHandler, bindingResolver,macroRepo);
importedParser.parse(incl,false,macroMap);
importedParser.parse(incl);
Map<String, String> macros = macroRepo.get(incl);
if (macros!=null)
macroMap.putAll(macros);
@ -88,10 +73,14 @@ public class CdtCppFileParser extends CppFileParser {
}
macroRepo.putMacros(fileFullPath,macroMap,tu.getMacroDefinitions());
tu.accept(bridge);
fileEntity = entityRepo.getEntity(fileFullPath);
((FileEntity)fileEntity).cacheAllExpressions();
bridge.done();
return;
}
@Override
protected boolean isPhase2Files(String fileFullPath) {
if (fileFullPath.endsWith(".h") || fileFullPath.endsWith(".hh") || fileFullPath.endsWith(".hpp")
|| fileFullPath.endsWith(".hxx"))
return true;
return false;
}
}

4
src/main/java/depends/extractor/cpp/cdt/CppVisitor.java
View File

@ -335,8 +335,4 @@ public class CppVisitor extends ASTVisitor {
}
return super.visit(parameterDeclaration);
}
public void done() {
context.done();
}
}

7
src/main/java/depends/extractor/golang/GoFileParser.java
View File

@ -28,7 +28,6 @@ import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
@ -41,8 +40,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.io.IOException;
public class GoFileParser implements depends.extractor.FileParser{
private EntityRepo entityRepo;
public class GoFileParser extends depends.extractor.FileParser{
private IBindingResolver bindingResolver;
public GoFileParser(EntityRepo entityRepo, IBindingResolver bindingResolver) {
this.entityRepo = entityRepo;
@ -50,8 +48,7 @@ public class GoFileParser implements depends.extractor.FileParser{
}
@Override
public void parse(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
public void parseFile(String fileFullPath) throws IOException {
CharStream input = CharStreams.fromFileName(fileFullPath);
Lexer lexer = new GoLexer(input);
lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), lexer.getInterpreter().decisionToDFA, new PredictionContextCache()));

15
src/main/java/depends/extractor/java/JavaFileParser.java
View File

@ -24,11 +24,9 @@ SOFTWARE.
package depends.extractor.java;
import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import depends.extractor.FileParser;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
@ -41,8 +39,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.io.IOException;
public class JavaFileParser implements depends.extractor.FileParser{
private EntityRepo entityRepo;
public class JavaFileParser extends FileParser {
private IBindingResolver bindingResolver;
public JavaFileParser(EntityRepo entityRepo, IBindingResolver bindingResolver) {
this.entityRepo = entityRepo;
@ -50,8 +47,7 @@ public class JavaFileParser implements depends.extractor.FileParser{
}
@Override
public void parse(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
public void parseFile(String fileFullPath) throws IOException {
CharStream input = CharStreams.fromFileName(fileFullPath);
Lexer lexer = new JavaLexer(input);
lexer.setInterpreter(new LexerATNSimulator(lexer, lexer.getATN(), lexer.getInterpreter().decisionToDFA, new PredictionContextCache()));
@ -63,11 +59,8 @@ public class JavaFileParser implements depends.extractor.FileParser{
ParseTreeWalker walker = new ParseTreeWalker();
try {
walker.walk(bridge, parser.compilationUnit());
Entity fileEntity = entityRepo.getEntity(fileFullPath);
((FileEntity)fileEntity).cacheAllExpressions();
interpreter.clearDFA();
bridge.done();
}catch (Exception e) {
System.err.println("error encountered during parse..." );
e.printStackTrace();

5
src/main/java/depends/extractor/java/JavaListener.java
View File

@ -334,9 +334,4 @@ public class JavaListener extends JavaParserBaseListener {
context.currentType().addTypeParameter(GenericName.build(typeParam.IDENTIFIER().getText()));
}
}
public void done() {
context.done();
}
}

6
src/main/java/depends/extractor/kotlin/KotlinFileParser.java
View File

@ -3,7 +3,6 @@ package depends.extractor.kotlin;
import depends.entity.repo.EntityRepo;
import depends.extractor.FileParser;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
@ -12,11 +11,10 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.io.IOException;
public class KotlinFileParser implements FileParser {
public class KotlinFileParser extends FileParser {
@Override
public void parse(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
public void parseFile(String fileFullPath) throws IOException {
CharStream input = CharStreams.fromFileName(fileFullPath);
Lexer lexer = new KotlinLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);

19
src/main/java/depends/extractor/pom/PomFileParser.java
View File

@ -24,14 +24,11 @@ SOFTWARE.
package depends.extractor.pom;
import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import depends.extractor.FileParser;
import depends.extractor.xml.XMLLexer;
import depends.extractor.xml.XMLParser;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.antlr.v4.runtime.CommonTokenStream;
@ -41,9 +38,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.io.IOException;
import java.util.List;
public class PomFileParser implements FileParser {
private EntityRepo entityRepo;
public class PomFileParser extends FileParser {
private PomProcessor parseCreator;
private List<String> includePaths;
private IBindingResolver bindingResolver;
@ -56,14 +51,7 @@ public class PomFileParser implements FileParser {
}
@Override
public void parse(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
/* If file already exist, skip it */
Entity fileEntity = entityRepo.getEntity(fileFullPath);
if (fileEntity!=null && fileEntity instanceof FileEntity) {
return;
}
/*parse file*/
public void parseFile(String fileFullPath) throws IOException {
CharStream input = CharStreams.fromFileName(fileFullPath);
Lexer lexer = new XMLLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
@ -71,9 +59,6 @@ public class PomFileParser implements FileParser {
PomListener bridge = new PomListener(fileFullPath, entityRepo, includePaths,parseCreator, bindingResolver);
ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(bridge, parser.document());
fileEntity = entityRepo.getEntity(fileFullPath);
bridge.done();
((FileEntity)fileEntity).cacheAllExpressions();
}
}

5
src/main/java/depends/extractor/pom/PomListener.java
View File

@ -162,9 +162,4 @@ public class PomListener extends XMLParserBaseListener {
String name = p.Name().get(0).getText();
return name;
}
public void done() {
context.done();
}
}

3
src/main/java/depends/extractor/python/union/PythonCodeListener.java
View File

@ -368,7 +368,4 @@ public class PythonCodeListener extends PythonParserBaseListener{
expressionUsage.stopExpr();
super.exitAssert_stmt(ctx);
}
public void done() {
context.done();
}
}

17
src/main/java/depends/extractor/python/union/PythonFileParser.java
View File

@ -1,12 +1,10 @@
package depends.extractor.python.union;
import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import depends.extractor.FileParser;
import depends.extractor.IncludedFileLocator;
import depends.extractor.python.PythonLexer;
import depends.extractor.python.PythonParser;
import depends.extractor.IncludedFileLocator;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.antlr.v4.runtime.CharStream;
@ -17,8 +15,7 @@ import org.antlr.v4.runtime.tree.ParseTreeWalker;
import java.io.IOException;
public class PythonFileParser implements FileParser {
private EntityRepo entityRepo;
public class PythonFileParser extends FileParser {
private IBindingResolver bindingResolver;
private IncludedFileLocator includeFileLocator;
private PythonProcessor processor;
@ -32,13 +29,8 @@ public class PythonFileParser implements FileParser {
}
@Override
public void parse(String fileFullPath) throws IOException {
public void parseFile(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
/** If file already exist, skip it */
Entity fileEntity = entityRepo.getEntity(fileFullPath);
if (fileEntity!=null && fileEntity instanceof FileEntity) {
return;
}
CharStream input = CharStreams.fromFileName(fileFullPath);
Lexer lexer = new PythonLexer(input);
CommonTokenStream tokens = new CommonTokenStream(lexer);
@ -48,9 +40,6 @@ public class PythonFileParser implements FileParser {
PythonCodeListener bridge = new PythonCodeListener(fileFullPath, entityRepo, bindingResolver, includeFileLocator, processor);
ParseTreeWalker walker = new ParseTreeWalker();
walker.walk(bridge, parser.file_input());
fileEntity = entityRepo.getEntity(fileFullPath);
((FileEntity)fileEntity).cacheAllExpressions();
bridge.done();
}
}

20
src/main/java/depends/extractor/ruby/jruby/JRubyFileParser.java
View File

@ -24,14 +24,11 @@ SOFTWARE.
package depends.extractor.ruby.jruby;
import depends.entity.Entity;
import depends.entity.FileEntity;
import depends.entity.repo.EntityRepo;
import depends.extractor.FileParser;
import depends.extractor.IncludedFileLocator;
import depends.extractor.ParserCreator;
import depends.relations.IBindingResolver;
import multilang.depends.util.file.FileUtil;
import org.antlr.v4.runtime.CharStream;
import org.antlr.v4.runtime.CharStreams;
import org.jrubyparser.CompatVersion;
@ -41,8 +38,7 @@ import org.jrubyparser.parser.ParserConfiguration;
import java.io.IOException;
import java.io.StringReader;
public class JRubyFileParser implements FileParser {
private EntityRepo entityRepo;
public class JRubyFileParser extends FileParser {
private IncludedFileLocator includesFileLocator;
private IBindingResolver bindingResolver;
private ParserCreator parserCreator;
@ -58,26 +54,16 @@ public class JRubyFileParser implements FileParser {
@SuppressWarnings("unchecked")
@Override
public void parse(String fileFullPath) throws IOException {
fileFullPath = FileUtil.uniqFilePath(fileFullPath);
/** If file already exist, skip it */
Entity fileEntity = entityRepo.getEntity(fileFullPath);
if (fileEntity!=null && fileEntity instanceof FileEntity) {
return;
}
public void parseFile(String fileFullPath) throws IOException {
CharStream input = CharStreams.fromFileName(fileFullPath);
Parser rubyParser = new Parser();
StringReader in = new StringReader(input.toString());
CompatVersion version = CompatVersion.RUBY2_3;
ParserConfiguration config = new ParserConfiguration(0, version);
try {
Node node = rubyParser.parse("<code>", in, config);
Node node = rubyParser.parse(fileFullPath, in, config);
JRubyVisitor parser = new JRubyVisitor(fileFullPath, entityRepo, includesFileLocator, bindingResolver,parserCreator);
node.accept(parser);
fileEntity = entityRepo.getEntity(fileFullPath);
((FileEntity)fileEntity).cacheAllExpressions();
parser.done();
}catch(Exception e) {
System.err.println("parsing error in "+fileFullPath);
}

3
src/main/java/depends/extractor/ruby/jruby/JRubyVisitor.java
View File

@ -277,7 +277,4 @@ public class JRubyVisitor extends NoopVisitor {
return super.visit(node);
}
public void done() {
context.done();
}
}

2
src/test/java/depends/extractor/cpp/ParseErrorTest.java
View File

@ -37,7 +37,7 @@ public class ParseErrorTest extends CppParserTest{
Map<String, String> macroMap = new HashMap<>();
macroMap.put("AP_DECLARE(x)","x");
parser.parse(src,true,macroMap);
parser.parse(src,macroMap);
}
resolveAllBindings();
}