p47035891
/
depends-public
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

support python dot include

This commit is contained in:
Gang ZHANG 2019-06-02 19:14:54 +08:00
parent 151b7b90e4
commit 5ece4e7c6b
8 changed files with 64 additions and 20 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
lib_install.sh
View File

@ -1,4 +1,4 @@
mvn install:install-file -DgroupId=org.jruby -DartifactId=org.jruby.jrubyparser -Dversion=0.5.5-SNAPSHOT -Dpackaging=jar -Dfile=./jars/jrubyparser-0.5.5-SNAPSHOT.jar -DgeneratePom=true
mvn install:install-file -DgroupId=org.jruby -DartifactId=jrubyparser -Dversion=0.5.5-SNAPSHOT -Dpackaging=jar -Dfile=./jars/jrubyparser-0.5.5-SNAPSHOT.jar -DgeneratePom=true
mvn install:install-file -DgroupId=org.eclipse.cdt -DartifactId=org.eclipse.cdt.core -Dversion=6.5.0.201806170908 -Dpackaging=jar -Dfile=./jars/org.eclipse.cdt.core-6.5.0.201806170908.jar -DgeneratePom=true

3
src/main/antlr4/depends/extractor/python/Python3.g4
View File

@ -186,7 +186,7 @@ raise_stmt: 'raise' (test ('from' test)?)?;
import_stmt: import_name | import_from;
import_name: 'import' dotted_as_names;
// note below: the ('.' | '...') is necessary because '...' is tokenized as ELLIPSIS
import_from: ('from' (('.' | '...')* dotted_name | ('.' | '...')+)
import_from: ('from' (dot_or_ellipsis* dotted_name | dot_or_ellipsis+)
'import' ('*' | '(' import_as_names ')' | import_as_names));
import_as_name: NAME ('as' NAME)?;
dotted_as_name: dotted_name ('as' NAME)?;
@ -196,6 +196,7 @@ dotted_name: NAME ('.' NAME)*;
global_stmt: 'global' NAME (',' NAME)*;
nonlocal_stmt: 'nonlocal' NAME (',' NAME)*;
assert_stmt: 'assert' test (',' test)?;
dot_or_ellipsis: ('.' | '...');
compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated | async_stmt;
async_stmt: ASYNC (funcdef | with_stmt | for_stmt);

50
src/main/java/depends/extractor/python/Python3CodeListener.java
View File

@ -19,6 +19,7 @@ import depends.entity.repo.EntityRepo;
import depends.extractor.python.Python3Parser.ClassdefContext;
import depends.extractor.python.Python3Parser.DecoratedContext;
import depends.extractor.python.Python3Parser.DecoratorContext;
import depends.extractor.python.Python3Parser.Dot_or_ellipsisContext;
import depends.extractor.python.Python3Parser.Dotted_as_nameContext;
import depends.extractor.python.Python3Parser.FuncdefContext;
import depends.extractor.python.Python3Parser.Global_stmtContext;
@ -63,8 +64,16 @@ public class Python3CodeListener extends Python3BaseListener {
@Override
public void enterImport_from(Import_fromContext ctx) {
String moduleName = ctx.dotted_name().getText();
String fullName = foundImportedModuleOrPackage(moduleName);
String moduleName = null;
if (ctx.dotted_name()!=null) {
moduleName = ctx.dotted_name().getText();
}
int prefixDotCount = 0;
for (Dot_or_ellipsisContext item:ctx.dot_or_ellipsis()) {
prefixDotCount +=item.getText().length();
}
String fullName = foundImportedModuleOrPackage(prefixDotCount,moduleName);
if (fullName!=null) {
if (ctx.import_as_names()==null) {//import *
ContainerEntity moduleEntity = (ContainerEntity)(entityRepo.getEntity(fullName));
@ -104,27 +113,40 @@ public class Python3CodeListener extends Python3BaseListener {
if (ctx.NAME()!=null) {
aliasedName = ctx.NAME().getText();
}
String fullName = foundImportedModuleOrPackage(originalName);
String fullName = foundImportedModuleOrPackage(0,originalName);
context.foundNewImport(new NameAliasImport(fullName,entityRepo.getEntity(fullName),aliasedName));
super.enterDotted_as_name(ctx);
}
private String foundImportedModuleOrPackage(String originalName) {
String importedName = originalName.replace(".", File.separator);
String fullName = includeFileLocator.uniqFileName(context.currentFile().getRawName(), importedName);
if (fullName==null) {
fullName = includeFileLocator.uniqFileName(context.currentFile().getRawName(), importedName+".py");
private String foundImportedModuleOrPackage(int prefixDotCount, String originalName) {
String dir = FileUtil.getLocatedDir(context.currentFile().getRawName());
String preFix = "";
for (int i=0;i<prefixDotCount-1;i++) {
preFix = preFix + ".." + File.separator;
}
dir = dir + preFix;
String fullName = null;
if (originalName!=null) {
String importedName = originalName.replace(".", File.separator);
fullName = includeFileLocator.uniqFileName(dir, importedName);
if (fullName==null) {
fullName = includeFileLocator.uniqFileName(dir, importedName+".py");
}
}else {
fullName = FileUtil.uniqFilePath(dir);
}
if (fullName!=null) {
if (FileUtil.isDirectory(fullName)) {
File d = new File(fullName);
File[] files = d.listFiles();
for (File file:files) {
if (!file.isDirectory()) {
if (file.getAbsolutePath().endsWith(".py")) {
visitIncludedFile(FileUtil.uniqFilePath(file.getAbsolutePath()));
if (!FileUtil.uniqFilePath(fullName).equals(FileUtil.uniqFilePath(dir))){
File d = new File(fullName);
File[] files = d.listFiles();
for (File file:files) {
if (!file.isDirectory()) {
if (file.getAbsolutePath().endsWith(".py")) {
visitIncludedFile(FileUtil.uniqFilePath(file.getAbsolutePath()));
}
}
}
}

5
src/main/java/depends/extractor/ruby/IncludedFileLocator.java
View File

@ -34,10 +34,9 @@ public class IncludedFileLocator {
public IncludedFileLocator(List<String> includedPath) {
this.includesPath = includedPath;
}
public String uniqFileName(String startLocation, String importedFilename) {
public String uniqFileName(String dirPath, String importedFilename) {
if (FileUtil.existFile(importedFilename)) return FileUtil.uniqFilePath(importedFilename);
if (startLocation!=null) {
String dirPath = FileUtil.getLocatedDir(startLocation);
if (dirPath!=null) {
String path = dirPath + File.separator + importedFilename;
if (FileUtil.existFile(path)) return FileUtil.uniqFilePath(path);
}

4
src/main/java/depends/extractor/ruby/RubyHandlerContext.java
View File

@ -36,6 +36,7 @@ import depends.extractor.HandlerContext;
import depends.extractor.ParserCreator;
import depends.importtypes.FileImport;
import depends.relations.Inferer;
import depends.util.FileUtil;
public class RubyHandlerContext extends HandlerContext {
@ -65,7 +66,8 @@ public class RubyHandlerContext extends HandlerContext {
if(methodName.equals("require") || methodName.equals("require_relative")) {
for (String importedFilename:params) {
if (!importedFilename.endsWith(".rb")) importedFilename = importedFilename + ".rb";
String inclFileName = includedFileLocator.uniqFileName(currentFile().getRawName(),importedFilename);
String dir = FileUtil.getLocatedDir(currentFile().getRawName());
String inclFileName = includedFileLocator.uniqFileName(dir,importedFilename);
if (inclFileName==null) {
System.err.println("Warning: cannot found included file " + importedFilename );
continue;

16
src/test/java/depends/extractor/python/PythonImportTest.java
View File

@ -118,4 +118,20 @@ public class PythonImportTest extends PythonParserTest {
this.assertContainsRelation(file, DependencyType.CALL,"foo");
this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1]));
}
@Test
public void should_parse_import_with_prefix_dots() throws IOException {
String[] srcs = new String[] {
"./src/test/resources/python-code-examples/import_with_dir/importing.py",
"./src/test/resources/python-code-examples/import_with_dir/imported_a.py",
};
for (String src:srcs) {
PythonFileParser parser = createParser(src);
parser.parse();
}
inferer.resolveAllBindings();
Entity file = repo.getEntity(FileUtil.uniqFilePath(srcs[0]));
this.assertContainsRelation(file, DependencyType.IMPORT,FileUtil.uniqFilePath(srcs[1]));
}
}

2
src/test/resources/python-code-examples/import_with_dir/imported_a.py Normal file
View File

@ -0,0 +1,2 @@
def foo():
pass

2
src/test/resources/python-code-examples/import_with_dir/importing.py Normal file
View File

@ -0,0 +1,2 @@
from .imported_a import foo
from . import imported_a