diff --git a/com.ibm.wala.cast.python.jython.test/.classpath b/com.ibm.wala.cast.python.jython.test/.classpath index fa387882a..11925f62e 100644 --- a/com.ibm.wala.cast.python.jython.test/.classpath +++ b/com.ibm.wala.cast.python.jython.test/.classpath @@ -17,11 +17,11 @@ + - @@ -29,7 +29,6 @@ - diff --git a/com.ibm.wala.cast.python.jython.test/.launchers/TestCalls.launch b/com.ibm.wala.cast.python.jython.test/.launchers/TestCalls.launch deleted file mode 100644 index 93ad55bbe..000000000 --- a/com.ibm.wala.cast.python.jython.test/.launchers/TestCalls.launch +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython.test/.launchers/TestTensorflowModel.launch b/com.ibm.wala.cast.python.jython.test/.launchers/TestTensorflowModel.launch deleted file mode 100644 index 291093c6b..000000000 --- a/com.ibm.wala.cast.python.jython.test/.launchers/TestTensorflowModel.launch +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython.test/pom.xml b/com.ibm.wala.cast.python.jython.test/pom.xml index da415893c..e239f4e07 100644 --- a/com.ibm.wala.cast.python.jython.test/pom.xml +++ b/com.ibm.wala.cast.python.jython.test/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -26,27 +26,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT junit diff --git a/com.ibm.wala.cast.python.jython/data/annots1.py b/com.ibm.wala.cast.python.jython/data/annots1.py new file mode 100644 index 000000000..85900b1a7 --- /dev/null +++ b/com.ibm.wala.cast.python.jython/data/annots1.py @@ -0,0 +1,3 @@ +def greet(name: str, age: int) -> str: + print('Hello {0}, you are {1} years old'.format(name, age)) + diff --git a/com.ibm.wala.cast.python.jython/data/except1.py b/com.ibm.wala.cast.python.jython/data/except1.py new file mode 100644 index 000000000..29f73752d --- /dev/null +++ b/com.ibm.wala.cast.python.jython/data/except1.py @@ -0,0 +1,14 @@ +import sys + +try: + f = open('myfile.txt') + s = f.readline() + i = int(s.strip()) +except IOError as e: + print "I/O error:" + print "{0}: {1}".format(e.errno, e.strerror) +except ValueError: + print "Could not convert data to an integer." +except: + print "Unexpected error:", sys.exc_info()[0] + raise diff --git a/com.ibm.wala.cast.python.jython/pom.xml b/com.ibm.wala.cast.python.jython/pom.xml index 26028a96c..ec3a737ac 100644 --- a/com.ibm.wala.cast.python.jython/pom.xml +++ b/com.ibm.wala.cast.python.jython/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -11,27 +11,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala @@ -40,8 +40,8 @@ org.python - jython - 2.7.2a1+ + jython-slim + 2.7.3a1-SNAPSHOT commons-cli diff --git a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java index f05018121..407392fa0 100644 --- a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java +++ b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java @@ -11,6 +11,7 @@ package com.ibm.wala.cast.python.loader; import java.io.IOException; +import java.util.List; import org.python.core.PyObject; @@ -29,6 +30,7 @@ import com.ibm.wala.cast.tree.rewrite.PatternBasedRewriter; import com.ibm.wala.cast.util.CAstPattern.Segments; import com.ibm.wala.classLoader.IClassLoader; +import com.ibm.wala.classLoader.Module; import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.classLoader.SourceModule; import com.ibm.wala.ipa.cha.IClassHierarchy; @@ -43,8 +45,8 @@ public Python2Loader(IClassHierarchy cha) { } @Override - protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M) throws IOException { - RewritingTranslatorToCAst x = new RewritingTranslatorToCAst(M, new PythonModuleParser((SourceModule)M, typeDictionary) { + protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M, List allModules) throws IOException { + RewritingTranslatorToCAst x = new RewritingTranslatorToCAst(M, new PythonModuleParser((SourceModule)M, typeDictionary, allModules) { @Override public CAstEntity translateToCAst() throws Error, IOException { CAstEntity ce = super.translateToCAst(); diff --git a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java index 206b457b0..2d9392b2f 100644 --- a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java +++ b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java @@ -14,6 +14,8 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; +import java.util.Collections; +import java.util.List; import org.antlr.runtime.ANTLRInputStream; import org.antlr.runtime.CharStream; @@ -21,6 +23,7 @@ import com.ibm.wala.cast.tree.CAstEntity; import com.ibm.wala.cast.tree.impl.CAstTypeDictionaryImpl; import com.ibm.wala.cast.util.CAstPrinter; +import com.ibm.wala.classLoader.Module; import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.classLoader.SourceModule; import com.ibm.wala.classLoader.SourceURLModule; @@ -38,7 +41,7 @@ protected WalaPythonParser makeParser() throws IOException { return new WalaPythonParser(file, fileName.getName(), "UTF-8"); } - public PythonModuleParser(SourceModule fileName, CAstTypeDictionaryImpl types) { + public PythonModuleParser(SourceModule fileName, CAstTypeDictionaryImpl types, List allModules) { super(types); this.fileName = fileName; } @@ -50,7 +53,7 @@ protected String scriptName() { public static void main(String[] args) throws Exception { URL url = new URL(args[0]); - PythonParser p = new PythonModuleParser(new SourceURLModule(url), new CAstTypeDictionaryImpl()); + PythonParser p = new PythonModuleParser(new SourceURLModule(url), new CAstTypeDictionaryImpl(), Collections.singletonList(new SourceURLModule(url))); CAstEntity script = p.translateToCAst(); System.err.println(script); System.err.println(CAstPrinter.print(script)); diff --git a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java index 819a0a725..144384d45 100644 --- a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java +++ b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java @@ -118,7 +118,7 @@ import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.HashSetFactory; import com.ibm.wala.util.collections.ReverseIterator; -import com.ibm.wala.util.warnings.Warning; +import com.ibm.wala.core.util.warnings.Warning; abstract public class PythonParser extends AbstractParser implements TranslatorToCAst { @@ -232,7 +232,7 @@ private Position makePosition(PythonTree p) { int last_col; int last_line = p.getLineno() + lines.length - 1; if ("".equals(s) || lines.length <= 1) { - last_col = p.getCharPositionInLine() + (p.getCharStopIndex() - p.getCharStartIndex()); + last_col = p.getCol_offset() + (p.getCharStopIndex() - p.getCharStartIndex()); } else { assert (lines.length > 1); last_col = lines[lines.length-1].length(); @@ -262,7 +262,7 @@ public int getFirstLine() { @Override public int getFirstCol() { - return p.getCharPositionInLine(); + return p.getCol_offset(); } @Override @@ -1092,7 +1092,7 @@ public CAstNode visitImport(Import arg0) throws Exception { int i = 0; CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size() ]; for(alias n : arg0.getInternalNames()) { - CAstNode obj = importAst(n.getInternalNameNodes()); + CAstNode obj = importAst(arg0, n.getInternalNameNodes()); elts[i++] = notePosition(Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any)), obj != null? @@ -1109,7 +1109,7 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { elts[0] = Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(tree, PythonCAstToIRTranslator.Any)), - importAst(arg0.getInternalModuleNames())); + importAst(arg0, arg0.getInternalModuleNames())); int i = 1; for(alias n : arg0.getInternalNames()) { @@ -1123,10 +1123,12 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { return Ast.makeNode(CAstNode.BLOCK_STMT, elts); } - private CAstNode importAst(java.util.List names ) { + private final boolean wholeStatement = true; + + private CAstNode importAst(R importNode, java.util.List names ) { CAstNode importAst = notePosition(Ast.makeNode(CAstNode.PRIMITIVE, Ast.makeConstant("import"), - Ast.makeConstant(names.get(0).getInternalId())), names.get(0)); + Ast.makeConstant(names.get(0).getInternalId())), wholeStatement? importNode: names.get(0)); for(int i = 1; i < names.size(); i++) { importAst = notePosition(Ast.makeNode(CAstNode.OBJECT_REF, importAst, diff --git a/com.ibm.wala.cast.python.jython3.test/.classpath b/com.ibm.wala.cast.python.jython3.test/.classpath index fa387882a..11925f62e 100644 --- a/com.ibm.wala.cast.python.jython3.test/.classpath +++ b/com.ibm.wala.cast.python.jython3.test/.classpath @@ -17,11 +17,11 @@ + - @@ -29,7 +29,6 @@ - diff --git a/com.ibm.wala.cast.python.jython3.test/.launchers/TestCalls.launch b/com.ibm.wala.cast.python.jython3.test/.launchers/TestCalls.launch deleted file mode 100644 index 93ad55bbe..000000000 --- a/com.ibm.wala.cast.python.jython3.test/.launchers/TestCalls.launch +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3.test/.launchers/TestTensorflowModel.launch b/com.ibm.wala.cast.python.jython3.test/.launchers/TestTensorflowModel.launch deleted file mode 100644 index 291093c6b..000000000 --- a/com.ibm.wala.cast.python.jython3.test/.launchers/TestTensorflowModel.launch +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3.test/pom.xml b/com.ibm.wala.cast.python.jython3.test/pom.xml index 918723aa8..46ed32d28 100644 --- a/com.ibm.wala.cast.python.jython3.test/pom.xml +++ b/com.ibm.wala.cast.python.jython3.test/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -26,27 +26,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT junit diff --git a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- calls.launch b/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- calls.launch deleted file mode 100644 index fefc4f4b1..000000000 --- a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- calls.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- except.launch b/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- except.launch deleted file mode 100644 index 79d267bbb..000000000 --- a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- except.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- georgiana.launch b/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- georgiana.launch deleted file mode 100644 index 505d3e437..000000000 --- a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- georgiana.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- mnist.launch b/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- mnist.launch deleted file mode 100644 index cd31f65a4..000000000 --- a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- mnist.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- tf.launch b/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- tf.launch deleted file mode 100644 index e1bead80c..000000000 --- a/com.ibm.wala.cast.python.jython3/.launchers/PythonDriver- tf.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.jython3/.settings/org.eclipse.jdt.core.prefs b/com.ibm.wala.cast.python.jython3/.settings/org.eclipse.jdt.core.prefs index 8b5c4dca7..cac0df4df 100644 --- a/com.ibm.wala.cast.python.jython3/.settings/org.eclipse.jdt.core.prefs +++ b/com.ibm.wala.cast.python.jython3/.settings/org.eclipse.jdt.core.prefs @@ -1,12 +1,7 @@ eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.methodParameters=do not generate org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 -org.eclipse.jdt.core.compiler.codegen.unusedLocal=preserve org.eclipse.jdt.core.compiler.compliance=1.8 -org.eclipse.jdt.core.compiler.debug.lineNumber=generate -org.eclipse.jdt.core.compiler.debug.localVariable=generate -org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error diff --git a/com.ibm.wala.cast.python.jython3/pom.xml b/com.ibm.wala.cast.python.jython3/pom.xml index 9d962bca3..da24ca446 100644 --- a/com.ibm.wala.cast.python.jython3/pom.xml +++ b/com.ibm.wala.cast.python.jython3/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -11,27 +11,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/client/PytestAnalysisEngine.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/client/PytestAnalysisEngine.java index 28ba671c9..fe593070b 100644 --- a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/client/PytestAnalysisEngine.java +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/client/PytestAnalysisEngine.java @@ -19,7 +19,7 @@ import com.ibm.wala.ssa.SSAAbstractInvokeInstruction; import com.ibm.wala.util.CancelException; import com.ibm.wala.util.intset.OrdinalSet; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PytestAnalysisEngine extends PythonAnalysisEngine { diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/driver/PytestDriver.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/driver/PytestDriver.java new file mode 100644 index 000000000..5ac1e56e7 --- /dev/null +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/driver/PytestDriver.java @@ -0,0 +1,22 @@ +package com.ibm.wala.cast.python.driver; + +import java.io.IOException; + +import com.ibm.wala.cast.python.client.PytestAnalysisEngine; +import com.ibm.wala.ipa.callgraph.propagation.PropagationCallGraphBuilder; +import com.ibm.wala.util.CancelException; + +public class PytestDriver extends Driver { + + public static void main(String[] args) throws IOException, CancelException { + PytestAnalysisEngine E = new PytestAnalysisEngine() { + @Override + public Void performAnalysis(PropagationCallGraphBuilder builder) throws CancelException { + return null; + } + }; + + new PytestDriver().runit(E, args); + } + +} diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/PytestLoader.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/PytestLoader.java index 8b6146372..0ee877ca7 100644 --- a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/PytestLoader.java +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/PytestLoader.java @@ -18,13 +18,14 @@ import com.ibm.wala.cast.tree.visit.CAstVisitor; import com.ibm.wala.classLoader.CallSiteReference; import com.ibm.wala.classLoader.IClass; +import com.ibm.wala.classLoader.ModuleEntry; +import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.ipa.cha.IClassHierarchy; import com.ibm.wala.types.FieldReference; import com.ibm.wala.types.TypeName; import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; public class PytestLoader extends Python3Loader { @@ -37,8 +38,8 @@ public PytestLoader(IClassHierarchy cha) { } @Override - protected TranslatorToIR initTranslator() { - return new PythonCAstToIRTranslator(this) { + protected TranslatorToIR initTranslator(Set> topLevelEntities) { + return new PythonCAstToIRTranslator(this, topLevelEntities) { private boolean isPytestEntry(CAstEntity F) { if (F.getType() instanceof CAstType.Function) { diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3Loader.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3Loader.java index 1ea4b99f3..cc13ba0fd 100644 --- a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3Loader.java +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/loader/Python3Loader.java @@ -11,6 +11,7 @@ package com.ibm.wala.cast.python.loader; import java.io.IOException; +import java.util.List; import org.python.core.PyObject; @@ -29,6 +30,7 @@ import com.ibm.wala.cast.tree.rewrite.PatternBasedRewriter; import com.ibm.wala.cast.util.CAstPattern.Segments; import com.ibm.wala.classLoader.IClassLoader; +import com.ibm.wala.classLoader.Module; import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.classLoader.SourceModule; import com.ibm.wala.ipa.cha.IClassHierarchy; @@ -43,8 +45,8 @@ public Python3Loader(IClassHierarchy cha) { } @Override - protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M) throws IOException { - RewritingTranslatorToCAst x = new RewritingTranslatorToCAst(M, new PythonModuleParser((SourceModule)M, typeDictionary) { + protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M, List allModules) throws IOException { + RewritingTranslatorToCAst x = new RewritingTranslatorToCAst(M, new PythonModuleParser((SourceModule)M, typeDictionary, allModules) { @Override public CAstEntity translateToCAst() throws Error, IOException { CAstEntity ce = super.translateToCAst(); diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java index 206b457b0..17a30ef9d 100644 --- a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java @@ -14,19 +14,35 @@ import java.io.InputStreamReader; import java.io.Reader; import java.net.URL; +import java.util.Collections; +import java.util.List; +import java.util.Optional; +import java.util.Set; +import java.util.function.Consumer; +import java.util.stream.Collector; +import java.util.stream.Collectors; import org.antlr.runtime.ANTLRInputStream; import org.antlr.runtime.CharStream; +import org.python.antlr.ast.ImportFrom; +import com.ibm.wala.cast.python.ir.PythonCAstToIRTranslator; import com.ibm.wala.cast.tree.CAstEntity; +import com.ibm.wala.cast.tree.CAstNode; +import com.ibm.wala.cast.tree.impl.CAstSymbolImpl; import com.ibm.wala.cast.tree.impl.CAstTypeDictionaryImpl; import com.ibm.wala.cast.util.CAstPrinter; +import com.ibm.wala.classLoader.FileModule; +import com.ibm.wala.classLoader.Module; import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.classLoader.SourceModule; import com.ibm.wala.classLoader.SourceURLModule; +import com.ibm.wala.util.collections.HashSetFactory; public class PythonModuleParser extends PythonParser { + private final Set localModules= HashSetFactory.make(); + private final SourceModule fileName; protected URL getParsedURL() throws IOException { @@ -38,19 +54,78 @@ protected WalaPythonParser makeParser() throws IOException { return new WalaPythonParser(file, fileName.getName(), "UTF-8"); } - public PythonModuleParser(SourceModule fileName, CAstTypeDictionaryImpl types) { + @Override + protected PythonParser.CAstVisitor makeVisitor(WalkContext context, WalaPythonParser parser) { + return new CAstVisitor(context, parser) { + + @Override + public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { + Optional s = arg0.getInternalModuleNames().stream() + .map(n -> { return n.getInternalId(); }) + .reduce((a, b) -> { return a + "/" + b; }); + if (s.isPresent()) { + String moduleName = s.get(); + if (! localModules.contains(moduleName + ".py")) { + moduleName = s.get() + "/__init__"; + } + if (localModules.contains(moduleName + ".py")) { + String yuck = moduleName; + return Ast.makeNode(CAstNode.BLOCK_STMT, + arg0.getInternalNames().stream() + .map(a -> a.getInternalName()) + .map(n -> Ast.makeNode(CAstNode.DECL_STMT, + Ast.makeConstant(new CAstSymbolImpl(n, PythonCAstToIRTranslator.Any)), + Ast.makeNode(CAstNode.PRIMITIVE, + Ast.makeConstant("import"), + Ast.makeConstant(yuck), + Ast.makeConstant(n)))) + .collect(Collectors.toList())); + + } + } + + return super.visitImportFrom(arg0); + } + + }; + } + + public PythonModuleParser(SourceModule fileName, CAstTypeDictionaryImpl types, List allModules) { super(types); this.fileName = fileName; + allModules.forEach(m -> { + m.getEntries().forEachRemaining(new Consumer() { + @Override + public void accept(ModuleEntry f) { + if (f.isModuleFile()) { + f.asModule().getEntries().forEachRemaining(sm -> { + accept(sm); + }); + } else { + System.err.println("**CLS: " + scriptName((SourceModule)f)); + localModules.add(scriptName((SourceModule)f)); + } + } + }); + }); } @Override protected String scriptName() { - return fileName.getName(); + return scriptName(fileName); + } + + private static String scriptName(SourceModule fileName) { + if (fileName instanceof FileModule) { + return fileName.getClassName(); + } else { + return fileName.getName(); + } } public static void main(String[] args) throws Exception { URL url = new URL(args[0]); - PythonParser p = new PythonModuleParser(new SourceURLModule(url), new CAstTypeDictionaryImpl()); + PythonParser p = new PythonModuleParser(new SourceURLModule(url), new CAstTypeDictionaryImpl(), Collections.singletonList(new SourceURLModule(url))); CAstEntity script = p.translateToCAst(); System.err.println(script); System.err.println(CAstPrinter.print(script)); diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java index e30064f4c..34963b507 100644 --- a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java @@ -113,6 +113,7 @@ import com.ibm.wala.cast.tree.CAstEntity; import com.ibm.wala.cast.tree.CAstNode; import com.ibm.wala.cast.tree.CAstQualifier; +import com.ibm.wala.cast.tree.CAstSourcePositionMap; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; import com.ibm.wala.cast.tree.CAstType; import com.ibm.wala.cast.tree.impl.AbstractSourcePosition; @@ -129,7 +130,7 @@ import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.HashSetFactory; import com.ibm.wala.util.collections.ReverseIterator; -import com.ibm.wala.util.warnings.Warning; +import com.ibm.wala.core.util.warnings.Warning; abstract public class PythonParser extends AbstractParser implements TranslatorToCAst { @@ -151,44 +152,86 @@ public Collection getSupertypes() { interface WalkContext extends TranslatorToCAst.WalkContext { + default void addDefinedName(String name) { + getParent().addDefinedName(name); + } + + default WalkContext root() { + return getParent().root(); + } + WalkContext getParent(); default CAstEntity entity() { return getParent().entity(); } + default void addGlobal(String n) { + getParent().addGlobal(n); + } + } private static class RootContext extends TranslatorToCAst.RootContext implements WalkContext { private final Module ast; + private final java.util.Set downwardGlobals = HashSetFactory.make(); + private RootContext(Module ast) { this.ast = ast; } - + @Override public PythonTree top() { return ast; } + @Override public WalkContext getParent() { assert false; return null; } + + public void addGlobal(String g) { + downwardGlobals.add(g); + } + + private java.util.Set downwardGlobals() { + return downwardGlobals; + } + + public WalkContext root() { + return this; + } + + public void addDefinedName(String name) { + assert false; + } } private static class FunctionContext extends TranslatorToCAst.FunctionContext implements WalkContext { private final AbstractCodeEntity fun; + private final java.util.Set downwardGlobals; + private final java.util.Set definedNames = HashSetFactory.make(); public WalkContext getParent() { return parent; } - private FunctionContext(WalkContext parent, AbstractCodeEntity fun, PythonTree s) { + private FunctionContext(WalkContext parent, AbstractCodeEntity fun, java.util.Set downwardGlobals, PythonTree s) { super(parent, s); this.fun = fun; + this.downwardGlobals = downwardGlobals; } + public void addDefinedName(String name) { + definedNames.add(name); + } + + public void addGlobal(String g) { + downwardGlobals.add(g); + } + @Override public CAstEntity entity() { return fun; @@ -219,7 +262,7 @@ public Map> getScopedEntities() { return fun.getAllScopedEntities(); } } - + public class CAstVisitor extends AbstractParser.CAstVisitor implements VisitorIF { private final PythonParser.WalkContext context; private final WalaPythonParser parser; @@ -230,7 +273,7 @@ private CAstNode fail(PyObject tree) { return Ast.makeNode(CAstNode.EMPTY); } - private CAstVisitor(PythonParser.WalkContext context, WalaPythonParser parser) { + protected CAstVisitor(PythonParser.WalkContext context, WalaPythonParser parser) { this.context = context; this.parser = parser; } @@ -407,6 +450,20 @@ public Position getNamePosition() { } } + + for (int i = 0; i < nodes.size(); i++) { + CAstNode assign = nodes.get(i); + if (assign.getChild(0).getKind() == CAstNode.VAR) { + CAstNode nm = assign.getChild(0).getChild(0); + if (nm.getValue() instanceof String) { + context.addDefinedName((String)nm.getValue()); + nodes.set(i, Ast.makeNode(CAstNode.BLOCK_STMT, + Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl((String)nm.getValue(), CAstType.DYNAMIC))), + assign)); + } + } + } + return Ast.makeNode(CAstNode.BLOCK_EXPR, nodes.toArray(new CAstNode[nodes.size()])); } } @@ -539,7 +596,7 @@ public CAstNode visitClassDef(ClassDef arg0) throws Exception { public Collection getSupertypes() { Collection supertypes = HashSetFactory.make(); for(expr e : arg0.getInternalBases()) { - System.out.println(arg0.getInternalName() + " " + arg0.getType()+ " extends " + e.getText() + " " + e.getType()); +// System.out.println(arg0.getInternalName() + " " + arg0.getType()+ " extends " + e.getText() + " " + e.getType()); try { CAstType type = types.getCAstTypeFor(e.getText()); if (type != null) { @@ -789,7 +846,7 @@ public CAstNode visitDict(Dict arg0) throws Exception { args[i++] = keys.next().accept(this); args[i++] = vals.next().accept(this); } - return Ast.makeNode(CAstNode.OBJECT_LITERAL, args); + return notePosition(Ast.makeNode(CAstNode.OBJECT_LITERAL, args), arg0); } @@ -929,9 +986,10 @@ private CAstNode defineFunction(Str defaultCode = new CAstNode[ defaults.size() ]; for(expr dflt : defaults) { String name = functionName + "_default_" + arg; + context.root().addGlobal(name); defaultCode[arg] = - Ast.makeNode(CAstNode.DECL_STMT, - Ast.makeConstant(new CAstSymbolImpl(name, PythonCAstToIRTranslator.Any)), + Ast.makeNode(CAstNode.ASSIGN, + Ast.makeNode(CAstNode.VAR, Ast.makeConstant(name)), dflt.accept(this)); defaultVars[arg++] = Ast.makeNode(CAstNode.VAR, Ast.makeConstant(name)); } @@ -1022,15 +1080,17 @@ class PythonFunction extends PythonCodeType implements CAstType.Function { ai++; } - class PythonCodeEntity extends AbstractCodeEntity implements DynamicAnnotatableEntity { + class PythonCodeEntity extends AbstractCodeEntity implements PythonGlobalsEntity, DynamicAnnotatableEntity { + private final java.util.Set downwardGlobals; @Override public Iterable dynamicAnnotations() { return dynamicAnnotations; } - protected PythonCodeEntity(CAstType type) { + protected PythonCodeEntity(CAstType type, java.util.Set downwardGlobals) { super(type); + this.downwardGlobals = downwardGlobals; } @Override @@ -1114,11 +1174,18 @@ public Position getPosition(int arg) { public Position getNamePosition() { return namePos; } + + @Override + public java.util.Set downwardGlobals() { + return downwardGlobals; + } }; - PythonCodeEntity fun = new PythonCodeEntity(functionType); + java.util.Set downwardGlobals = HashSetFactory.make(); + + PythonCodeEntity fun = new PythonCodeEntity(functionType, downwardGlobals); - PythonParser.FunctionContext child = new PythonParser.FunctionContext(context, fun, function); + PythonParser.FunctionContext child = new PythonParser.FunctionContext(context, fun, downwardGlobals, function); CAstVisitor cv = new CAstVisitor(child, parser); for(S s : body) { nodes[i++] = s.accept(cv); @@ -1126,7 +1193,11 @@ public Position getNamePosition() { if (isMethod) { context.addScopedEntity(null, fun); - return null; + if (defaultCode.length == 0) { + return null; + } else { + return Ast.makeNode(CAstNode.BLOCK_EXPR, defaultCode); + } } else { CAstNode stmt = Ast.makeNode(CAstNode.FUNCTION_EXPR, Ast.makeConstant(fun)); @@ -1185,6 +1256,9 @@ public CAstNode visitGeneratorExp(GeneratorExp arg0) throws Exception { @Override public CAstNode visitGlobal(Global arg0) throws Exception { java.util.List internalNames = arg0.getInternalNameNodes(); + + internalNames.forEach(n -> context.addGlobal(n.getText())); + CAstNode[] x = new CAstNode[arg0.getInternalNameNodes().size()]; for(int i = 0; i < x.length; i++) x[i] = internalNames.get(i).accept(this); @@ -1204,18 +1278,18 @@ private CAstNode block(java.util.List block) throws Exception { @Override public CAstNode visitIf(If arg0) throws Exception { - return Ast.makeNode(CAstNode.IF_STMT, + return notePosition(Ast.makeNode(CAstNode.IF_STMT, arg0.getInternalTest().accept(this), block(arg0.getInternalBody()), - block(arg0.getInternalOrelse())); + block(arg0.getInternalOrelse())), arg0); } @Override public CAstNode visitIfExp(IfExp arg0) throws Exception { - return Ast.makeNode(CAstNode.IF_EXPR, + return notePosition(Ast.makeNode(CAstNode.IF_EXPR, arg0.getInternalTest().accept(this), arg0.getInternalBody().accept(this), - arg0.getInternalOrelse().accept(this)); + arg0.getInternalOrelse().accept(this)), arg0); } private String name(alias n) { @@ -1231,7 +1305,7 @@ public CAstNode visitImport(Import arg0) throws Exception { int i = 0; CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size() ]; for(alias n : arg0.getInternalNames()) { - CAstNode obj = importAst(n.getInternalNameNodes()); + CAstNode obj = importAst(arg0, n.getInternalNameNodes()); elts[i++] = notePosition(Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any)), obj != null? @@ -1248,7 +1322,7 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { elts[0] = notePosition(Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(tree, PythonCAstToIRTranslator.Any)), - importAst(arg0.getInternalModuleNames())), arg0); + importAst(arg0, arg0.getInternalModuleNames())), arg0); int i = 1; for(alias n : arg0.getInternalNames()) { @@ -1268,10 +1342,12 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception { return Ast.makeNode(CAstNode.BLOCK_STMT, elts); } - private CAstNode importAst(java.util.List names ) { + private final boolean wholeStatement = true; + + private CAstNode importAst(T importNode, java.util.List names ) { CAstNode importAst = notePosition(Ast.makeNode(CAstNode.PRIMITIVE, Ast.makeConstant("import"), - Ast.makeConstant(names.get(0).getInternalId())), names.get(0)); + Ast.makeConstant(names.get(0).getInternalId())), wholeStatement? importNode: names.get(0)); for(int i = 1; i < names.size(); i++) { importAst = notePosition(Ast.makeNode(CAstNode.OBJECT_REF, importAst, @@ -1475,10 +1551,18 @@ private CAstNode doGenerators(java.util.List generators, CAstNode Ast.makeNode(CAstNode.EACH_ELEMENT_GET, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(tempName)), c.getInternalTarget().accept(this))))); - - result = Ast.makeNode(CAstNode.BLOCK_EXPR, + + result = notePosition(Ast.makeNode(CAstNode.BLOCK_EXPR, + Ast.makeNode(CAstNode.DECL_STMT, Ast.makeConstant(new CAstSymbolImpl(tempName, PythonCAstToIRTranslator.Any)), - c.getInternalIter().accept(this)), + c.getInternalIter().accept(this)), + + Ast.makeNode(CAstNode.ASSIGN, + c.getInternalTarget().accept(this), + Ast.makeNode(CAstNode.EACH_ELEMENT_GET, + Ast.makeNode(CAstNode.VAR, Ast.makeConstant(tempName)), + Ast.makeConstant(null))), + Ast.makeNode(CAstNode.LOOP, test, Ast.makeNode(CAstNode.BLOCK_EXPR, @@ -1487,7 +1571,7 @@ private CAstNode doGenerators(java.util.List generators, CAstNode Ast.makeNode(CAstNode.OBJECT_REF, Ast.makeNode(CAstNode.VAR, Ast.makeConstant(tempName)), c.getInternalTarget().accept(this))), - result))); + result))), c); } @@ -1502,7 +1586,7 @@ public CAstNode visitModule(Module arg0) throws Exception { for(PythonTree c : arg0.getChildren()) { elts.add(c.accept(this)); } - return Ast.makeNode(CAstNode.BLOCK_EXPR, elts.toArray(new CAstNode[ elts.size() ])); + return Ast.makeNode(CAstNode.BLOCK_STMT, elts.toArray(new CAstNode[ elts.size() ])); } else { return Ast.makeNode(CAstNode.EMPTY); } @@ -1560,9 +1644,9 @@ public CAstNode visitRaise(Raise arg0) throws Exception { @Override public CAstNode visitReturn(Return arg0) throws Exception { if(arg0.getInternalValue() == null) - return Ast.makeNode(CAstNode.RETURN, Ast.makeNode(CAstNode.VAR, Ast.makeConstant("None"))); + return notePosition(Ast.makeNode(CAstNode.RETURN, Ast.makeNode(CAstNode.VAR, Ast.makeConstant("None"))), arg0); else - return Ast.makeNode(CAstNode.RETURN, arg0.getInternalValue().accept(this)); + return notePosition(Ast.makeNode(CAstNode.RETURN, arg0.getInternalValue().accept(this)), arg0); } @Override @@ -1938,6 +2022,10 @@ public , K extends CopyKey> void addRewriter(CAst } + protected CAstVisitor makeVisitor(PythonParser.WalkContext context, WalaPythonParser parser) { + return new CAstVisitor(context, parser); + } + @Override public CAstEntity translateToCAst() throws Error, IOException { WalaPythonParser parser = makeParser(); @@ -1976,19 +2064,19 @@ public Collection getSupertypes() { }; - WalkContext root = new PythonParser.RootContext(pythonAst); - CAstEntity script = new AbstractScriptEntity(scriptName(), scriptType) { + RootContext root = new PythonParser.RootContext(pythonAst); + + class PythonScriptEntity extends AbstractScriptEntity implements PythonGlobalsEntity { + private PythonScriptEntity(String file, CAstType type) throws Exception { + super(file, type); + context = new PythonParser.FunctionContext(root, this, root.downwardGlobals(), pythonAst); + visitor = makeVisitor(context, parser); + cast = pythonAst.accept(visitor); + } private final WalkContext context; private final CAstVisitor visitor; - private final CAstNode cast; - - { - context = new PythonParser.FunctionContext(root, this, pythonAst); - visitor = new CAstVisitor(context, parser); - cast = pythonAst.accept(visitor); - } - + private final CAstNode cast; @Override public CAstNode getAST() { @@ -2000,6 +2088,7 @@ public Position getPosition() { return visitor.makePosition(pythonAst); } + @Override public Position getPosition(int arg) { return null; } @@ -2009,9 +2098,14 @@ public Position getNamePosition() { return null; } + @Override + public java.util.Set downwardGlobals() { + return root.downwardGlobals(); + } + }; - return script; + return new PythonScriptEntity(scriptName(), scriptType); } catch (Exception e) { throw new Error(Collections.singleton(new Warning(Warning.SEVERE) { @Override diff --git a/com.ibm.wala.cast.python.ml.j2ee/pom.xml b/com.ibm.wala.cast.python.ml.j2ee/pom.xml index bb126a7d3..e473507ce 100644 --- a/com.ibm.wala.cast.python.ml.j2ee/pom.xml +++ b/com.ibm.wala.cast.python.ml.j2ee/pom.xml @@ -53,27 +53,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala diff --git a/com.ibm.wala.cast.python.ml.test/.classpath b/com.ibm.wala.cast.python.ml.test/.classpath index e7cb7269f..483b5a436 100644 --- a/com.ibm.wala.cast.python.ml.test/.classpath +++ b/com.ibm.wala.cast.python.ml.test/.classpath @@ -2,11 +2,11 @@ + - @@ -14,7 +14,6 @@ - diff --git a/com.ibm.wala.cast.python.ml.test/pom.xml b/com.ibm.wala.cast.python.ml.test/pom.xml index a5d5cbef8..db90064f6 100644 --- a/com.ibm.wala.cast.python.ml.test/pom.xml +++ b/com.ibm.wala.cast.python.ml.test/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -31,27 +31,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT junit diff --git a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestPandasModel.java b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestPandasModel.java index 516d563ec..ebd332441 100644 --- a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestPandasModel.java +++ b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestPandasModel.java @@ -30,7 +30,7 @@ public void testPandas1() throws ClassHierarchyException, IllegalArgumentExcepti Map> excelTableFields = PandasReadExcelAnalysis.readExcelAnalysis(CG, PA, H); - System.out.println(excelTableFields); +// System.out.println(excelTableFields); boolean foundDfqol = false; boolean foundDfdemog = false; diff --git a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflowModel.java b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflowModel.java index 3c6eced45..d2ddd9254 100644 --- a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflowModel.java +++ b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflowModel.java @@ -6,8 +6,13 @@ import org.junit.Test; +import com.ibm.wala.cast.ipa.callgraph.CAstCallGraphUtil; +import com.ibm.wala.cast.python.client.PythonAnalysisEngine; +import com.ibm.wala.cast.python.ipa.callgraph.PythonSSAPropagationCallGraphBuilder; +import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.CallGraph; +import com.ibm.wala.ipa.callgraph.propagation.SSAPropagationCallGraphBuilder; import com.ibm.wala.ipa.cha.ClassHierarchyException; import com.ibm.wala.util.CancelException; @@ -15,7 +20,14 @@ public class TestTensorflowModel extends TestPythonMLCallGraphShape { @Test public void testTf1() throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException { - CallGraph CG = process("tf1.py"); + PythonAnalysisEngine E = makeEngine("tf1.py"); + PythonSSAPropagationCallGraphBuilder builder = E.defaultCallGraphBuilder(); + CallGraph CG = builder.makeCallGraph(builder.getOptions()); + +// CAstCallGraphUtil.AVOID_DUMP = false; +// CAstCallGraphUtil.dumpCG(((SSAPropagationCallGraphBuilder)builder).getCFAContextInterpreter(), builder.getPointerAnalysis(), CG); + +// System.err.println(CG); Collection nodes = getNodes(CG, "script tf1.py/model_fn"); assert ! nodes.isEmpty() : "model_fn should be called"; diff --git a/com.ibm.wala.cast.python.ml/.classpath b/com.ibm.wala.cast.python.ml/.classpath index 4c4d4d87f..fa7d3f985 100644 --- a/com.ibm.wala.cast.python.ml/.classpath +++ b/com.ibm.wala.cast.python.ml/.classpath @@ -20,17 +20,17 @@ - + - + diff --git a/com.ibm.wala.cast.python.ml/data/tensorflow.xml b/com.ibm.wala.cast.python.ml/data/tensorflow.xml index aca285d83..ed1d4eb29 100644 --- a/com.ibm.wala.cast.python.ml/data/tensorflow.xml +++ b/com.ibm.wala.cast.python.ml/data/tensorflow.xml @@ -341,18 +341,19 @@ - + + value="model"/> diff --git a/com.ibm.wala.cast.python.ml/pom.xml b/com.ibm.wala.cast.python.ml/pom.xml index 38e24fcb3..f83685479 100644 --- a/com.ibm.wala.cast.python.ml/pom.xml +++ b/com.ibm.wala.cast.python.ml/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -16,27 +16,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala diff --git a/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/analysis/PandasReadExcelAnalysis.java b/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/analysis/PandasReadExcelAnalysis.java index d3782d9e7..f900f7b3f 100644 --- a/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/analysis/PandasReadExcelAnalysis.java +++ b/com.ibm.wala.cast.python.ml/source/com/ibm/wala/cast/python/ml/analysis/PandasReadExcelAnalysis.java @@ -4,12 +4,14 @@ import java.util.Map; import java.util.Set; +import com.ibm.wala.cast.ir.ssa.AstPropertyRead; import com.ibm.wala.cast.python.ssa.PythonInstructionVisitor; import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.types.AstMethodReference; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.CallGraph; +import com.ibm.wala.ipa.callgraph.propagation.ConstantKey; import com.ibm.wala.ipa.callgraph.propagation.HeapModel; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis; @@ -41,6 +43,24 @@ public void visitPythonInvoke(PythonInvokeInstruction inst) { } }); ir.visitAllInstructions(new PythonInstructionVisitor() { + + @Override + public void visitPropertyRead(AstPropertyRead instruction) { + PA.getPointsToSet(H.getPointerKeyForLocal(script, instruction.getObjectRef())).forEach((InstanceKey obj) -> { + if (excelTableFields.containsKey(obj)) { + if (excelTableFields.containsKey(obj)) { + PA.getPointsToSet(H.getPointerKeyForLocal(script, instruction.getMemberRef())).forEach((InstanceKey field) -> { + if (field instanceof ConstantKey && + ((ConstantKey)field).getValue() instanceof String) + { + excelTableFields.get(obj).add(((ConstantKey)field).getValue()); + } + }); + } + } + }); + } + @Override public void visitGet(SSAGetInstruction instruction) { PA.getPointsToSet(H.getPointerKeyForLocal(script, instruction.getRef())).forEach((InstanceKey obj) -> { diff --git a/com.ibm.wala.cast.python.test/.launchers/TestCalls.launch b/com.ibm.wala.cast.python.test/.launchers/TestCalls.launch deleted file mode 100644 index d357530da..000000000 --- a/com.ibm.wala.cast.python.test/.launchers/TestCalls.launch +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.test/.launchers/TestTensorflowModel.launch b/com.ibm.wala.cast.python.test/.launchers/TestTensorflowModel.launch deleted file mode 100644 index 291093c6b..000000000 --- a/com.ibm.wala.cast.python.test/.launchers/TestTensorflowModel.launch +++ /dev/null @@ -1,20 +0,0 @@ - - - - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python.test/data/annotations2.py b/com.ibm.wala.cast.python.test/data/annotations2.py index f8c04c94f..e6cc0f48e 100644 --- a/com.ibm.wala.cast.python.test/data/annotations2.py +++ b/com.ibm.wala.cast.python.test/data/annotations2.py @@ -102,12 +102,12 @@ def test_class(class_): def test_cols(columns): print("(" + str(columns) + ")") -@pytest.mark.parametrize("repeats, kwargs, error, msg",[ -(2, dict(axis=1), ValueError, "'axis"), -(-1, dict(), ValueError, "negative"), -([1, 2], dict(), ValueError, "shape")]) -def test_dict(repeats, kwargs, error, msg): - print("(" + str(repeats) + " " + str(kwargs) + " " + str(error) + " " + msg + ")") +#@pytest.mark.parametrize("repeats, kwargs, error, msg",[ +#(2, dict(axis=1), ValueError, "'axis"), +#(-1, dict(), ValueError, "negative"), +#([1, 2], dict(), ValueError, "shape")]) +#def test_dict(repeats, kwargs, error, msg): +# print("(" + str(repeats) + " " + str(kwargs) + " " + str(error) + " " + msg + ")") from pandas_shim import DataFrame import np_shim as np diff --git a/com.ibm.wala.cast.python.test/data/cmp2.py b/com.ibm.wala.cast.python.test/data/cmp2.py new file mode 100644 index 000000000..6a9511074 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/cmp2.py @@ -0,0 +1,6 @@ +def cin(a, b): + return a in b + +x = {} +x["f"] = 5 +cin("f", x) diff --git a/com.ibm.wala.cast.python.test/data/contro1.py b/com.ibm.wala.cast.python.test/data/contro1.py new file mode 100644 index 000000000..fc642ba40 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/contro1.py @@ -0,0 +1,20 @@ +import sys + +class e1(Exception): + def __str__(self): + return "exception" + +a = sys.argv[1] +b = sys.argv[2] + +try: + if a == b: + raise e1 + + a = 7 + +except e1 as e: + a = e + +print(a) +reveal_type(a) diff --git a/com.ibm.wala.cast.python.test/data/control2.py b/com.ibm.wala.cast.python.test/data/control2.py new file mode 100644 index 000000000..390dc4207 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/control2.py @@ -0,0 +1,17 @@ +import sys + +a = sys.argv[1] +b = sys.argv[2] + +i = 0 +x = 1 +while i < 5: + if a == b: + x = "foo" + break + + x = 7 + i = i + 1 + +print(x) +reveal_type(x) diff --git a/com.ibm.wala.cast.python.test/data/control3.py b/com.ibm.wala.cast.python.test/data/control3.py new file mode 100644 index 000000000..ec8a90484 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/control3.py @@ -0,0 +1,12 @@ +import sys +a = sys.argv[1] +b = sys.argv[2] +#i = 0 +for i in range(0,5): #i < 5: + if a == b: + x = "foo" + break + x = 7 + #i = i + 1 +print(x) +reveal_type(x) diff --git a/com.ibm.wala.cast.python.test/data/control4.py b/com.ibm.wala.cast.python.test/data/control4.py new file mode 100644 index 000000000..b74432825 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/control4.py @@ -0,0 +1,22 @@ +import sys +a = sys.argv[1] +b = sys.argv[2] + +class c1: + a + +x = c1() +x.a = "foo" +#reveal_type(x.a) + +y = c1() +y.a = 7 +#reveal_type(y.a) + +if a == b: + y = x + +y.a = 7 + +print(x.a) +reveal_type(x.a) diff --git a/com.ibm.wala.cast.python.test/pom.xml b/com.ibm.wala.cast.python.test/pom.xml index e735cffd9..0e34419b6 100644 --- a/com.ibm.wala.cast.python.test/pom.xml +++ b/com.ibm.wala.cast.python.test/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -16,33 +16,33 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT test-fixtures com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT junit diff --git a/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestCalls.java b/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestCalls.java index 54af5e7f0..51d8e8be5 100644 --- a/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestCalls.java +++ b/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestCalls.java @@ -128,7 +128,18 @@ public void testCalls5() throws ClassHierarchyException, IllegalArgumentExceptio @Test public void testCalls6() throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException { - CallGraph CG = process("calls6.py"); + PythonAnalysisEngine e = new PythonAnalysisEngine() { + @Override + public Void performAnalysis(PropagationCallGraphBuilder builder) throws CancelException { + assert false; + return null; + } + }; + e.setModuleFiles(Collections.singleton(getScript("calls6.py"))); + PropagationCallGraphBuilder cgBuilder = (PropagationCallGraphBuilder) e.defaultCallGraphBuilder(); + CallGraph CG = cgBuilder.makeCallGraph(cgBuilder.getOptions()); + CAstCallGraphUtil.AVOID_DUMP = false; + CAstCallGraphUtil.dumpCG((SSAContextInterpreter)cgBuilder.getContextInterpreter(), cgBuilder.getPointerAnalysis(), CG); verifyGraphAssertions(CG, assertionsCalls6); } @@ -183,8 +194,8 @@ public Void performAnalysis(PropagationCallGraphBuilder builder) throws CancelEx e.setModuleFiles(Collections.singleton(getScript("defaultValuesTest.py"))); PropagationCallGraphBuilder cgBuilder = (PropagationCallGraphBuilder) e.defaultCallGraphBuilder(); CallGraph CG = cgBuilder.makeCallGraph(cgBuilder.getOptions()); - //CAstCallGraphUtil.AVOID_DUMP = false; - //CAstCallGraphUtil.dumpCG((SSAContextInterpreter)cgBuilder.getContextInterpreter(), cgBuilder.getPointerAnalysis(), CG); + CAstCallGraphUtil.AVOID_DUMP = false; + CAstCallGraphUtil.dumpCG((SSAContextInterpreter)cgBuilder.getContextInterpreter(), cgBuilder.getPointerAnalysis(), CG); verifyGraphAssertions(CG, assertionsDefaultValues); } } diff --git a/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestPythonCallGraphShape.java b/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestPythonCallGraphShape.java index a13d83080..ffe6d4299 100644 --- a/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestPythonCallGraphShape.java +++ b/com.ibm.wala.cast.python.test/source/com/ibm/wala/cast/python/test/TestPythonCallGraphShape.java @@ -29,7 +29,7 @@ import com.ibm.wala.util.CancelException; import com.ibm.wala.util.NullProgressMonitor; import com.ibm.wala.util.collections.HashSetFactory; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public abstract class TestPythonCallGraphShape extends TestCallGraphShape { diff --git a/com.ibm.wala.cast.python/.launchers/PythonDriver- calls.launch b/com.ibm.wala.cast.python/.launchers/PythonDriver- calls.launch deleted file mode 100644 index fefc4f4b1..000000000 --- a/com.ibm.wala.cast.python/.launchers/PythonDriver- calls.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python/.launchers/PythonDriver- except.launch b/com.ibm.wala.cast.python/.launchers/PythonDriver- except.launch deleted file mode 100644 index 79d267bbb..000000000 --- a/com.ibm.wala.cast.python/.launchers/PythonDriver- except.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python/.launchers/PythonDriver- georgiana.launch b/com.ibm.wala.cast.python/.launchers/PythonDriver- georgiana.launch deleted file mode 100644 index 505d3e437..000000000 --- a/com.ibm.wala.cast.python/.launchers/PythonDriver- georgiana.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python/.launchers/PythonDriver- mnist.launch b/com.ibm.wala.cast.python/.launchers/PythonDriver- mnist.launch deleted file mode 100644 index cd31f65a4..000000000 --- a/com.ibm.wala.cast.python/.launchers/PythonDriver- mnist.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python/.launchers/PythonDriver- tf.launch b/com.ibm.wala.cast.python/.launchers/PythonDriver- tf.launch deleted file mode 100644 index e1bead80c..000000000 --- a/com.ibm.wala.cast.python/.launchers/PythonDriver- tf.launch +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - diff --git a/com.ibm.wala.cast.python/pom.xml b/com.ibm.wala.cast.python/pom.xml index ff0592bc7..a5bd93244 100644 --- a/com.ibm.wala.cast.python/pom.xml +++ b/com.ibm.wala.cast.python/pom.xml @@ -1,4 +1,4 @@ - + 4.0.0 com.ibm.wala @@ -11,27 +11,27 @@ com.ibm.wala com.ibm.wala.util - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.shrike - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.core - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT com.ibm.wala com.ibm.wala.cast.java - 1.5.6-SNAPSHOT + 1.5.8-SNAPSHOT org.eclipse.lsp4j diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/client/PythonAnalysisEngine.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/client/PythonAnalysisEngine.java index 4db91c937..e227ed015 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/client/PythonAnalysisEngine.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/client/PythonAnalysisEngine.java @@ -20,8 +20,10 @@ import com.ibm.wala.cast.python.ir.PythonLanguage; import com.ibm.wala.cast.python.loader.PythonLoaderFactory; import com.ibm.wala.cast.python.types.PythonTypes; +import com.ibm.wala.cast.python.util.PythonInterpreter; import com.ibm.wala.cast.types.AstMethodReference; import com.ibm.wala.cast.util.Util; +import com.ibm.wala.classLoader.FileModule; import com.ibm.wala.classLoader.IClass; import com.ibm.wala.classLoader.IClassLoader; import com.ibm.wala.classLoader.IField; @@ -30,6 +32,7 @@ import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.classLoader.SyntheticClass; import com.ibm.wala.client.AbstractAnalysisEngine; +import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.ipa.callgraph.AnalysisCacheImpl; import com.ibm.wala.ipa.callgraph.AnalysisOptions; import com.ibm.wala.ipa.callgraph.AnalysisScope; @@ -52,7 +55,7 @@ import com.ibm.wala.ipa.summaries.BypassMethodTargetSelector; import com.ibm.wala.ipa.summaries.BypassSyntheticClassLoader; import com.ibm.wala.ipa.summaries.XMLMethodSummaryReader; -import com.ibm.wala.shrikeBT.Constants; +import com.ibm.wala.shrike.shrikeBT.Constants; import com.ibm.wala.ssa.IRFactory; import com.ibm.wala.ssa.SSAOptions; import com.ibm.wala.ssa.SSAOptions.DefaultValues; @@ -67,11 +70,28 @@ import com.ibm.wala.util.WalaRuntimeException; import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.HashSetFactory; -import com.ibm.wala.util.strings.Atom; public abstract class PythonAnalysisEngine extends AbstractAnalysisEngine { + static { + try { + Class j3 = Class.forName("com.ibm.wala.cast.python.loader.Python3LoaderFactory"); + PythonAnalysisEngine.setLoaderFactory((Class) j3); + Class i3 = Class.forName("com.ibm.wala.cast.python.util.Python3Interpreter"); + PythonInterpreter.setInterpreter((PythonInterpreter)i3.newInstance()); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e) { + try { + Class j2 = Class.forName("com.ibm.wala.cast.python.loader.Python2LoaderFactory"); + PythonAnalysisEngine.setLoaderFactory((Class) j2); + Class i2 = Class.forName("com.ibm.wala.cast.python.util.Python2Interpreter"); + PythonInterpreter.setInterpreter((PythonInterpreter)i2.newInstance()); + } catch (ClassNotFoundException | InstantiationException | IllegalAccessException e1) { + assert false : e.getMessage() + ", then " + e1.getMessage(); + } + } + } + private static Class loaders; public static void setLoaderFactory(Class lf) { @@ -259,26 +279,19 @@ protected void addBypassLogic(IClassHierarchy cha, AnalysisOptions options) { addSummaryBypassLogic(options, "functools.xml"); } - private String scriptName(ModuleEntry m) { - String path = m.getName(); - return "Lscript " + (path.contains("/")? path.substring(path.lastIndexOf('/')+1): path); - } - + @Override protected Iterable makeDefaultEntrypoints(AnalysisScope scope, IClassHierarchy cha) { Set result = HashSetFactory.make(); - for(Module m : moduleFiles) { - m.getEntries().forEachRemaining(f -> { - IClass entry = cha.lookupClass(TypeReference.findOrCreate(PythonTypes.pythonLoader, TypeName.findOrCreate(scriptName(f)))); - assert entry != null: "bad root name " + scriptName(f) + ":\n" + cha; + cha.forEach(entry -> { + if (entry.getName().toString().endsWith(".py")) { MethodReference er = MethodReference.findOrCreate(entry.getReference(), AstMethodReference.fnSelector); result.add(new DefaultEntrypoint(er, cha)); - }); - } + } + }); return result; } - @Override protected PythonSSAPropagationCallGraphBuilder getCallGraphBuilder(IClassHierarchy cha, AnalysisOptions options, IAnalysisCacheView cache2) { IAnalysisCacheView cache = new AnalysisCacheImpl(irs, options.getSSAOptions()); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/driver/Driver.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/driver/Driver.java index c855a8eb0..a7a34e51b 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/driver/Driver.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/driver/Driver.java @@ -45,7 +45,7 @@ public class Driver { } } - protected static T runit(PythonAnalysisEngine E, String... args) + protected T runit(PythonAnalysisEngine E, String... args) throws IOException, CancelException { Set sources = HashSetFactory.make(); for(String file : args) { @@ -84,7 +84,7 @@ public Void performAnalysis(PropagationCallGraphBuilder builder) throws CancelEx } }; - runit(E, args); + new Driver().runit(E, args); } diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonConstructorTargetSelector.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonConstructorTargetSelector.java index 6a41bb481..513509ca2 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonConstructorTargetSelector.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonConstructorTargetSelector.java @@ -35,7 +35,7 @@ import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonConstructorTargetSelector implements MethodTargetSelector { private final Map ctors = HashMapFactory.make(); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonSSAPropagationCallGraphBuilder.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonSSAPropagationCallGraphBuilder.java index d39b32b79..5e213cdc8 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonSSAPropagationCallGraphBuilder.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonSSAPropagationCallGraphBuilder.java @@ -46,7 +46,7 @@ import com.ibm.wala.util.collections.Pair; import com.ibm.wala.util.intset.IntSetUtil; import com.ibm.wala.util.intset.MutableIntSet; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonSSAPropagationCallGraphBuilder extends AstSSAPropagationCallGraphBuilder { @@ -82,22 +82,20 @@ protected boolean sameMethod(CGNode opNode, String definingMethod) { private static final Collection types = Arrays.asList(PythonTypes.string, TypeReference.Int); + private final Map,BuiltinFunction> primitives = HashMapFactory.make(); + + public static class PythonConstraintVisitor extends AstConstraintVisitor implements PythonInstructionVisitor { + @Override + protected PythonSSAPropagationCallGraphBuilder getBuilder() { + return (PythonSSAPropagationCallGraphBuilder) builder; + } + public PythonConstraintVisitor(AstSSAPropagationCallGraphBuilder builder, CGNode node) { super(builder, node); } - private final Map,BuiltinFunction> primitives = HashMapFactory.make(); - - private BuiltinFunction ensure(Pair key) { - if (! primitives.containsKey(key)) { - primitives.put(key, new BuiltinFunction(this.getClassHierarchy(), key.fst, key.snd)); - } - - return primitives.get(key); - } - @Override public void visitGet(SSAGetInstruction instruction) { SymbolTable symtab = ir.getSymbolTable(); @@ -114,7 +112,7 @@ public void visitGet(SSAGetInstruction instruction) { for (InstanceKey ik : getInvariantContents(objVn)) { if (types.contains(ik.getConcreteType().getReference())) { Pair key = Pair.make(name, ik.getConcreteType().getReference()); - system.newConstraint(lvalKey, new ConcreteTypeKey(ensure(key))); + // system.newConstraint(lvalKey, new ConcreteTypeKey(getBuilder().ensure(key))); } } } else { @@ -126,7 +124,7 @@ public byte evaluate(PointsToSetVariable lhs, PointsToSetVariable[] rhs) { InstanceKey ik = system.getInstanceKey(i); if (types.contains(ik.getConcreteType().getReference())) { Pair key = Pair.make(name, ik.getConcreteType().getReference()); - system.newConstraint(lvalKey, new ConcreteTypeKey(ensure(key))); + // system.newConstraint(lvalKey, new ConcreteTypeKey(getBuilder().ensure(key))); } }); return NOT_CHANGED; @@ -183,7 +181,7 @@ protected void processCallingConstraints(CGNode caller, SSAAbstractInvokeInstruc // positional parameters PythonInvokeInstruction call = (PythonInvokeInstruction) instruction; - for(int i = 0; i < call.getNumberOfPositionalParameters() && i <= target.getMethod().getNumberOfParameters(); i++) { + for(int i = 0; i < call.getNumberOfPositionalParameters() && i < target.getMethod().getNumberOfParameters(); i++) { PointerKey lval = getPointerKeyForLocal(target, i+1); args.add(i); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonScopeMappingInstanceKeys.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonScopeMappingInstanceKeys.java index 84bd22b24..2176165dc 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonScopeMappingInstanceKeys.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonScopeMappingInstanceKeys.java @@ -10,6 +10,7 @@ import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.types.AstMethodReference; import com.ibm.wala.classLoader.IClass; +import com.ibm.wala.classLoader.IMethod; import com.ibm.wala.ipa.callgraph.CGNode; import com.ibm.wala.ipa.callgraph.propagation.InstanceKey; import com.ibm.wala.ipa.callgraph.propagation.InstanceKeyFactory; @@ -35,10 +36,11 @@ protected LexicalParent[] getParents(InstanceKey base) { cls = ((PythonInstanceMethodTrampoline)cls).getRealClass(); } - DynamicMethodObject function = (DynamicMethodObject) - cls.getMethod(AstMethodReference.fnSelector); + IMethod function = cls.getMethod(AstMethodReference.fnSelector); - return function==null? new LexicalParent[0]: function.getParents(); + return function==null || !(function instanceof DynamicMethodObject)? + new LexicalParent[0]: + ((DynamicMethodObject)function).getParents(); } @Override diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java index 9f8149136..b3cd3a43e 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java @@ -32,7 +32,7 @@ import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonTrampolineTargetSelector implements MethodTargetSelector { private final MethodTargetSelector base; @@ -66,10 +66,10 @@ public IMethod getCalleeTarget(CGNode caller, CallSiteReference site, IClass rec x.addStatement(PythonLanguage.Python.instructionFactory().GetInstruction(1, v1, 1, FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom("$self"), PythonTypes.Root))); int i = 0; - int[] params = new int[ Math.max(2, call.getNumberOfPositionalParameters()) ]; + int[] params = new int[ Math.max(2, call.getNumberOfPositionalParameters()+1) ]; params[i++] = v0; params[i++] = v1; - for(int j = 1; j < call.getNumberOfPositionalParameters()-1; j++) { + for(int j = 1; j < call.getNumberOfPositionalParameters(); j++) { params[i++] = j+1; } diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/BuiltinFunctions.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/BuiltinFunctions.java index eb2214f89..98401d24c 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/BuiltinFunctions.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/BuiltinFunctions.java @@ -28,7 +28,7 @@ import com.ibm.wala.types.TypeReference; import com.ibm.wala.types.annotations.Annotation; import com.ibm.wala.util.collections.HashMapFactory; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class BuiltinFunctions { @@ -90,19 +90,19 @@ public static class BuiltinFunction implements IClass { private final IMethod builtinCode; private final IClassHierarchy cha; - public BuiltinFunction(IClassHierarchy cha, String name, TypeReference returnedType) { + private BuiltinFunction(IClassHierarchy cha, String name, TypeReference returnedType) { this.cha = cha; this.ref = builtinFunction(name); this.builtinCode = returnedType==null? noopSummary(this, name): typeSummary(this, name, returnedType); } - public BuiltinFunction(IClassHierarchy cha, String name, int arg) { + private BuiltinFunction(IClassHierarchy cha, String name, int arg) { this.cha = cha; this.ref = builtinFunction(name); this.builtinCode = argSummary(this, name, arg); } - public BuiltinFunction(IClassHierarchy cha, String name) { + private BuiltinFunction(IClassHierarchy cha, String name) { this(cha, name, null); } @@ -267,7 +267,7 @@ public Collection getAnnotations() { } - public static TypeReference builtinFunction(String name) { + private static TypeReference builtinFunction(String name) { return TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lwala/builtin/" + name); } @@ -277,6 +277,7 @@ public static TypeReference builtinFunction(String name) { // builtinFunctions.put("enumerate", Either.forLeft(PythonTypes.enumerate)); builtinFunctions.put("enumerate", Either.forRight(2)); builtinFunctions.put("int", Either.forLeft(TypeReference.Int)); + builtinFunctions.put("round", Either.forLeft(TypeReference.Int)); builtinFunctions.put("len", Either.forLeft(TypeReference.Int)); builtinFunctions.put("list", Either.forLeft(PythonTypes.list)); builtinFunctions.put("range", Either.forLeft(PythonTypes.list)); @@ -286,6 +287,7 @@ public static TypeReference builtinFunction(String name) { builtinFunctions.put("type", Either.forLeft(PythonTypes.object)); builtinFunctions.put("zip", Either.forLeft(PythonTypes.list)); builtinFunctions.put("slice", Either.forRight(2)); + builtinFunctions.put("__delete__", Either.forRight(2)); } public static Set builtins() { diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonComprehensionTrampolines.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonComprehensionTrampolines.java index 431df2f5c..2ca202e1f 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonComprehensionTrampolines.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonComprehensionTrampolines.java @@ -17,7 +17,7 @@ import com.ibm.wala.types.Selector; import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonComprehensionTrampolines implements MethodTargetSelector { private final MethodTargetSelector base; diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonInstanceMethodTrampoline.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonInstanceMethodTrampoline.java index 9f3cf2284..94eff2c86 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonInstanceMethodTrampoline.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonInstanceMethodTrampoline.java @@ -20,7 +20,7 @@ import com.ibm.wala.types.FieldReference; import com.ibm.wala.types.TypeReference; import com.ibm.wala.types.annotations.Annotation; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonInstanceMethodTrampoline extends PythonSyntheticClass { private static final Atom selfName = Atom.findOrCreateUnicodeAtom("self"); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSuper.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSuper.java index 0c3aff258..7c70d3676 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSuper.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSuper.java @@ -48,7 +48,7 @@ import com.ibm.wala.util.intset.EmptyIntSet; import com.ibm.wala.util.intset.IntSet; import com.ibm.wala.util.intset.IntSetUtil; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonSuper { diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSyntheticClass.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSyntheticClass.java index 2daba3de2..e17ba18c9 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSyntheticClass.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/PythonSyntheticClass.java @@ -21,11 +21,11 @@ import com.ibm.wala.classLoader.IMethod; import com.ibm.wala.classLoader.SyntheticClass; import com.ibm.wala.ipa.cha.IClassHierarchy; -import com.ibm.wala.shrikeBT.Constants; +import com.ibm.wala.shrike.shrikeBT.Constants; import com.ibm.wala.types.Selector; import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.HashMapFactory; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonSyntheticClass extends SyntheticClass { protected final Map functions = HashMapFactory.make(); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/TurtleSummary.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/TurtleSummary.java index a9874730b..3b81a77ce 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/TurtleSummary.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/summaries/TurtleSummary.java @@ -30,8 +30,8 @@ import com.ibm.wala.ipa.callgraph.impl.AbstractRootMethod; import com.ibm.wala.ipa.cha.IClassHierarchy; import com.ibm.wala.ipa.summaries.BypassSyntheticClassLoader; -import com.ibm.wala.shrikeBT.IInvokeInstruction.Dispatch; -import com.ibm.wala.shrikeCT.InvalidClassFileException; +import com.ibm.wala.shrike.shrikeBT.IInvokeInstruction.Dispatch; +import com.ibm.wala.shrike.shrikeCT.InvalidClassFileException; import com.ibm.wala.ssa.SSAAbstractInvokeInstruction; import com.ibm.wala.types.Descriptor; import com.ibm.wala.types.FieldReference; @@ -42,7 +42,7 @@ import com.ibm.wala.types.annotations.Annotation; import com.ibm.wala.util.collections.HashSetFactory; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class TurtleSummary { public static final TypeReference turtleClassRef = TypeReference.findOrCreate(PythonTypes.pythonLoader, "Lturtle"); @@ -208,6 +208,22 @@ public Descriptor getDescriptor() { @Override public boolean hasLocalVariableTable() { return false; + } + + @Override + public boolean isAnnotation() { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean isEnum() { + return false; + } + + @Override + public boolean isModule() { + return false; } }; diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java index d8ee477b8..674d0c698 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonCAstToIRTranslator.java @@ -19,20 +19,22 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.Stack; import com.ibm.wala.cast.ir.ssa.AssignInstruction; import com.ibm.wala.cast.ir.ssa.AstGlobalRead; import com.ibm.wala.cast.ir.ssa.AstInstructionFactory; -import com.ibm.wala.cast.ir.translator.ArrayOpHandler; import com.ibm.wala.cast.ir.translator.AstTranslator; import com.ibm.wala.cast.loader.AstMethod.DebuggingInformation; import com.ibm.wala.cast.loader.DynamicCallSiteReference; import com.ibm.wala.cast.python.loader.DynamicAnnotatableEntity; import com.ibm.wala.cast.python.loader.PythonLoader; +import com.ibm.wala.cast.python.parser.AbstractParser.PythonGlobalsEntity; import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.tree.CAstEntity; import com.ibm.wala.cast.tree.CAstNode; +import com.ibm.wala.cast.tree.CAstSourcePositionMap; import com.ibm.wala.cast.tree.CAstSourcePositionMap.Position; import com.ibm.wala.cast.tree.CAstType; import com.ibm.wala.cast.tree.impl.CAstControlFlowRecorder; @@ -45,10 +47,12 @@ import com.ibm.wala.classLoader.CallSiteReference; import com.ibm.wala.classLoader.IClass; import com.ibm.wala.classLoader.IClassLoader; +import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.classLoader.NewSiteReference; -import com.ibm.wala.shrikeBT.IBinaryOpInstruction; -import com.ibm.wala.shrikeBT.IBinaryOpInstruction.IOperator; -import com.ibm.wala.shrikeBT.IInvokeInstruction.Dispatch; +import com.ibm.wala.core.util.strings.Atom; +import com.ibm.wala.shrike.shrikeBT.IBinaryOpInstruction; +import com.ibm.wala.shrike.shrikeBT.IBinaryOpInstruction.IOperator; +import com.ibm.wala.shrike.shrikeBT.IInvokeInstruction.Dispatch; import com.ibm.wala.ssa.SSAInstruction; import com.ibm.wala.ssa.SymbolTable; import com.ibm.wala.types.FieldReference; @@ -58,25 +62,17 @@ import com.ibm.wala.util.collections.HashMapFactory; import com.ibm.wala.util.collections.HashSetFactory; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; public class PythonCAstToIRTranslator extends AstTranslator { private final Map walaTypeNames = HashMapFactory.make(); private final Set> globalDeclSet = new HashSet<>(); private static boolean signleFileAnalysis = true; + private final Set> topLevelEntities; - public PythonCAstToIRTranslator(IClassLoader loader, Map namedEntityResolver, - ArrayOpHandler arrayOpHandler) { - super(loader, namedEntityResolver, arrayOpHandler); - } - - public PythonCAstToIRTranslator(IClassLoader loader, Map namedEntityResolver) { - super(loader, namedEntityResolver); - } - - public PythonCAstToIRTranslator(IClassLoader loader) { + public PythonCAstToIRTranslator(IClassLoader loader, Set> topLevelEntities) { super(loader); + this.topLevelEntities = topLevelEntities; } public static boolean isSingleFileAnalysis() { @@ -211,7 +207,9 @@ protected String composeEntityName(WalkContext parent, CAstEntity f) { @Override protected void doPrologue(WalkContext context) { if (context.currentScope().getEntity().getKind() == CAstEntity.SCRIPT_ENTITY) { - doGlobalWrite(context, context.currentScope().getEntity().getName(), PythonTypes.Root, 1); + context.cfg().unknownInstructions(() -> { + doGlobalWrite(context, context.currentScope().getEntity().getName(), PythonTypes.Root, 1); + }); } super.doPrologue(context); @@ -240,27 +238,27 @@ public void doArrayWrite(WalkContext context, int arrayValue, CAstNode arrayRef, @Override protected void doFieldRead(WalkContext context, int result, int receiver, CAstNode elt, CAstNode parent) { int currentInstruction = context.cfg().getCurrentInstruction(); - if (elt.getKind() == CAstNode.CONSTANT && elt.getValue() instanceof String) { - FieldReference f = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); - context.cfg().addInstruction(Python.instructionFactory().GetInstruction(currentInstruction, result, receiver, f)); - } else { +// if (elt.getKind() == CAstNode.CONSTANT && elt.getValue() instanceof String) { +// FieldReference f = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); +// context.cfg().addInstruction(Python.instructionFactory().GetInstruction(currentInstruction, result, receiver, f)); +// } else { visit(elt, context, this); assert context.getValue(elt) != -1; context.cfg().addInstruction(((AstInstructionFactory) insts).PropertyRead(currentInstruction, result, receiver, context.getValue(elt))); - } +// } context.cfg().noteOperands(currentInstruction, context.getSourceMap().getPosition(parent.getChild(0)), context.getSourceMap().getPosition(elt)); } @Override protected void doFieldWrite(WalkContext context, int receiver, CAstNode elt, CAstNode parent, int rval) { - if (elt.getKind() == CAstNode.CONSTANT && elt.getValue() instanceof String) { - FieldReference f = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); - context.cfg().addInstruction(Python.instructionFactory().PutInstruction(context.cfg().getCurrentInstruction(), receiver, rval, f)); - } else { +// if (elt.getKind() == CAstNode.CONSTANT && elt.getValue() instanceof String) { +// FieldReference f = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom((String)elt.getValue()), PythonTypes.Root); +// context.cfg().addInstruction(Python.instructionFactory().PutInstruction(context.cfg().getCurrentInstruction(), receiver, rval, f)); +// } else { visit(elt, context, this); assert context.getValue(elt) != -1; context.cfg().addInstruction(((AstInstructionFactory) insts).PropertyWrite(context.cfg().getCurrentInstruction(), receiver, context.getValue(elt), rval)); - } +// } } @Override @@ -270,7 +268,10 @@ protected void doMaterializeFunction(CAstNode node, WalkContext context, int res IClass cls = loader.lookupClass(TypeName.findOrCreate("L" + fnName)); TypeReference type = cls.getReference(); int idx = context.cfg().getCurrentInstruction(); - context.cfg().addInstruction(Python.instructionFactory().NewInstruction(idx, result, NewSiteReference.make(idx, type))); + + context.cfg().unknownInstructions(() -> { + context.cfg().addInstruction(Python.instructionFactory().NewInstruction(idx, result, NewSiteReference.make(idx, type))); + }); if (fn instanceof DynamicAnnotatableEntity) { if (((DynamicAnnotatableEntity)fn).dynamicAnnotations().iterator().hasNext()) { @@ -290,9 +291,11 @@ protected void doMaterializeFunction(CAstNode node, WalkContext context, int res } } - doGlobalWrite(context, fnName, PythonTypes.Root, result); - FieldReference fnField = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom(fn.getName()), PythonTypes.Root); - context.cfg().addInstruction(Python.instructionFactory().PutInstruction(context.cfg().getCurrentInstruction(), 1, result, fnField)); + context.cfg().unknownInstructions(() -> { + doGlobalWrite(context, fnName, PythonTypes.Root, result); + FieldReference fnField = FieldReference.findOrCreate(PythonTypes.Root, Atom.findOrCreateUnicodeAtom(fn.getName()), PythonTypes.Root); + context.cfg().addInstruction(Python.instructionFactory().PutInstruction(context.cfg().getCurrentInstruction(), 1, result, fnField)); + }); } @@ -313,37 +316,67 @@ protected void leaveFunctionEntity(CAstEntity n, WalkContext context, WalkContex } } + private final Stack globalsStack = new Stack<>(); + + @Override + protected boolean enterEntity(CAstEntity n, WalkContext context, CAstVisitor visitor) { + if (n.getOriginal() instanceof PythonGlobalsEntity) { + globalsStack.push((PythonGlobalsEntity)n.getOriginal()); + } + return super.enterEntity(n, context, visitor); + } + + @Override + protected void postProcessEntity(CAstEntity n, WalkContext context, CAstVisitor visitor) { + if (n.getOriginal() instanceof PythonGlobalsEntity) { + globalsStack.pop(); + } + super.postProcessEntity(n, context, visitor); + } + + private boolean isGlobalVar(String name) { + for(PythonGlobalsEntity e : globalsStack ) { + if (e.downwardGlobals().contains(name)) { + return true; + } + } + + return false; + } + @Override protected void leaveVar(CAstNode n, WalkContext c, CAstVisitor visitor) { WalkContext context = c; String nm = (String) n.getChild(0).getValue(); assert nm != null : "cannot find var for " + CAstPrinter.print(n, context.getSourceMap()); - Symbol s = context.currentScope().lookup(nm); - assert s != null : "cannot find symbol for " + nm + " at " + CAstPrinter.print(n, context.getSourceMap()); - assert s.type() != null : "no type for " + nm + " at " + CAstPrinter.print(n, context.getSourceMap()); - TypeReference type = makeType(s.type()); - if (context.currentScope().isGlobal(s) || isGlobal(context, nm)) { - c.setValue(n, doGlobalRead(n, context, nm, type)); - } else if (context.currentScope().isLexicallyScoped(s)) { - c.setValue(n, doLexicallyScopedRead(n, context, nm, type)); + + if ("0".equals(nm)) { + System.err.println("got here"); + } + + if (isGlobalVar(nm)) { + c.setValue(n, doGlobalRead(n, context, nm, PythonTypes.Root)); } else { - c.setValue(n, doLocalRead(context, nm, type)); + Symbol s = context.currentScope().lookup(nm); + assert s != null : "cannot find symbol for " + nm + " at " + CAstPrinter.print(n, context.getSourceMap()); + assert s.type() != null : "no type for " + nm + " at " + CAstPrinter.print(n, context.getSourceMap()); + TypeReference type = makeType(s.type()); + if (context.currentScope().isGlobal(s) || isGlobal(context, nm)) { + c.setValue(n, doGlobalRead(n, context, nm, type)); + } else if (context.currentScope().isLexicallyScoped(s)) { + c.setValue(n, doLexicallyScopedRead(n, context, nm, type)); + } else { + c.setValue(n, doLocalRead(context, nm, type)); + } } } -// @Override -// protected void leaveVarAssign(CAstNode n, CAstNode v, CAstNode a, WalkContext c, CAstVisitor visitor) { -// WalkContext context = c; -// int rval = c.getValue(v); -// String nm = (String) n.getChild(0).getValue(); -// Symbol ls = context.currentScope().lookup(nm); -// c.setValue(n, rval); -// assignValue(n, context, ls, nm, rval); -// } @Override protected void assignValue(CAstNode n, WalkContext context, Symbol ls, String nm, int rval) { - if (context.currentScope().isGlobal(ls) || isGlobal(context, nm)) + if (isGlobalVar(nm)) { + doGlobalWrite(context, nm, PythonTypes.Root, rval); + } else if (context.currentScope().isGlobal(ls) || isGlobal(context, nm)) doGlobalWrite(context, nm, makeType(ls.type()), rval); else if (context.currentScope().isLexicallyScoped(ls)) { doLexicallyScopedWrite(context, nm, makeType(ls.type()), rval); @@ -542,7 +575,7 @@ protected void doPrimitive(int resultVal, WalkContext context, CAstNode primitiv Position save = currentPosition; currentPosition = context.getSourceMap().getPosition(primitiveCall); - System.err.println("&&&&& switched to " + currentPosition); + // System.err.println("&&&&& switched to " + currentPosition); if (loader.lookupClass(TypeName.findOrCreate("Lscript " + name + ".py")) != null) { FieldReference global = makeGlobalRef("script " + name + ".py"); @@ -554,6 +587,16 @@ protected void doPrimitive(int resultVal, WalkContext context, CAstNode primitiv } currentPosition = save; + } else if (primitiveCall.getChildCount() == 3 && "import".equals(primitiveCall.getChild(0).getValue())) { + String scriptName = (String) primitiveCall.getChild(1).getValue(); + String eltName = (String) primitiveCall.getChild(2).getValue(); + + int idx = context.cfg().getCurrentInstruction(); + FieldReference global = makeGlobalRef("script " + scriptName + ".py"); + context.cfg().addInstruction(new AstGlobalRead(idx, resultVal, global)); + + idx = context.cfg().getCurrentInstruction(); + context.cfg().addInstruction(((AstInstructionFactory) insts).PropertyRead(idx, resultVal, resultVal, context.currentScope().getConstantValue(eltName))); } } @@ -655,6 +698,7 @@ void addGlobal(Scope scope,String varName){ Pair pair = Pair.make(scope,varName); globalDeclSet.add(pair); } + @Override protected boolean doVisit(CAstNode n, WalkContext context, CAstVisitor visitor) { if (n.getKind() == CAstNode.COMPREHENSION_EXPR) { @@ -713,4 +757,11 @@ protected Position[] getParameterPositions(CAstEntity e) { return ps; } + + @Override + protected void leaveBlockStmt(CAstNode n, WalkContext c, CAstVisitor visitor) { + // TODO Auto-generated method stub + super.leaveBlockStmt(n, c, visitor); + } + } diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java index 7fed0acf3..ab1317344 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ir/PythonLanguage.java @@ -51,8 +51,8 @@ import com.ibm.wala.ipa.modref.ExtendedHeapModel; import com.ibm.wala.ipa.modref.ModRef.ModVisitor; import com.ibm.wala.ipa.modref.ModRef.RefVisitor; -import com.ibm.wala.shrikeCT.BootstrapMethodsReader.BootstrapMethod; -import com.ibm.wala.shrikeCT.InvalidClassFileException; +import com.ibm.wala.shrike.shrikeCT.BootstrapMethodsReader.BootstrapMethod; +import com.ibm.wala.shrike.shrikeCT.InvalidClassFileException; import com.ibm.wala.ssa.SSAAbstractInvokeInstruction; import com.ibm.wala.ssa.SSAInstruction; import com.ibm.wala.ssa.SSAInvokeInstruction; @@ -61,7 +61,7 @@ import com.ibm.wala.types.TypeName; import com.ibm.wala.types.TypeReference; import com.ibm.wala.util.collections.Pair; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonLanguage implements Language { diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java index bba48eec1..eb8e4fc0b 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java @@ -2,7 +2,9 @@ import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Map; +import java.util.Set; import com.ibm.wala.cast.ir.translator.AstTranslator.AstLexicalInformation; import com.ibm.wala.cast.ir.translator.AstTranslator.WalkContext; @@ -30,6 +32,8 @@ import com.ibm.wala.classLoader.IField; import com.ibm.wala.classLoader.IMethod; import com.ibm.wala.classLoader.Language; +import com.ibm.wala.classLoader.Module; +import com.ibm.wala.classLoader.ModuleEntry; import com.ibm.wala.ipa.cha.IClassHierarchy; import com.ibm.wala.ssa.SSAInstruction; import com.ibm.wala.ssa.SSAInstructionFactory; @@ -41,10 +45,18 @@ import com.ibm.wala.types.TypeReference; import com.ibm.wala.types.annotations.Annotation; import com.ibm.wala.util.collections.HashSetFactory; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.util.collections.Pair; +import com.ibm.wala.core.util.strings.Atom; public abstract class PythonLoader extends CAstAbstractModuleLoader { + + @Override + public void init(List modules) { + // TODO Auto-generated method stub + super.init(modules); + } + public class DynamicMethodBody extends DynamicCodeBody { private final IClass container; @@ -144,8 +156,8 @@ protected boolean shouldTranslate(CAstEntity entity) { } @Override - protected TranslatorToIR initTranslator() { - return new PythonCAstToIRTranslator(this); + protected TranslatorToIR initTranslator(Set> topLevelEntities) { + return new PythonCAstToIRTranslator(this, topLevelEntities); } final CoreClass CodeBody = new CoreClass(PythonTypes.CodeBody.getName(), PythonTypes.rootTypeName, this, null); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/AbstractParser.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/AbstractParser.java index 18ab13802..0f0f4277b 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/AbstractParser.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/parser/AbstractParser.java @@ -13,15 +13,87 @@ public abstract class AbstractParser { + public interface PythonGlobalsEntity { + java.util.Set downwardGlobals(); + } + public static String[] defaultImportNames = new String[] { - "__name__", - "print", - "super", - "open", - "hasattr", "BaseException", + "DeprecationWarning", + "Exception", + "FutureWarning", + "NameError", + "None", + "RuntimeError", + "StopIteration", + "TypeError", + "UserWarning", + "ValueError", + "__doc__", + "__file__", + "__name__", "abs", + "all", + "any", + "bin", + "bool", + "bytes", + "callable", + "chr", + "complex", "del", + "dict", + "dir", + "divmod", + "eval", + "exec", + "exit", + "filter", + "float", + "format", + "frozenset", + "get_ipython", + "getattr", + "globals", + "hasattr", + "help", + "hex", + "id", + "input", + "isinstance", + "iter", + "locals", + "map", + "max", + "min", + "next", + "object", + "open", + "ord", + "pow", + "print", + "property", + "repr", + "reversed", + "set", + "super", + "tuple", + "vars", + + // names found in recent IPython + "NotImplementedError", + "Warning", + "cd", + "clear", + "pylab", + "RuntimeWarning", + "hist", + "matplotlib", + "recall", + "history", + "time", + "KeyError", + "display" }; protected final CAst Ast = new CAstImpl(); diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java index b97655295..1de3c5f5d 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java @@ -14,7 +14,7 @@ import com.ibm.wala.types.ClassLoaderReference; import com.ibm.wala.types.TypeName; import com.ibm.wala.types.TypeReference; -import com.ibm.wala.util.strings.Atom; +import com.ibm.wala.core.util.strings.Atom; public class PythonTypes extends AstTypeReference {