bug fixes for dataflow

wala · Sep 1, 2021 · 3cf9b35 · 3cf9b35
1 parent 3c3c7c0
commit 3cf9b35
Show file tree

Hide file tree

Showing 70 changed files with 753 additions and 549 deletions.
diff --git a/com.ibm.wala.cast.python.jython.test/.classpath b/com.ibm.wala.cast.python.jython.test/.classpath
@@ -17,19 +17,18 @@
 	</classpathentry>
 	<classpathentry kind="src" output="target/test-classes" path="test-source">
 		<attributes>
+			<attribute name="test" value="true"/>
 			<attribute name="gradle_scope" value="test"/>
 			<attribute name="gradle_used_by_scope" value="test"/>
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
-			<attribute name="test" value="true"/>
 		</attributes>
 	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>

diff --git a/com.ibm.wala.cast.python.jython.test/.launchers/TestCalls.launch b/com.ibm.wala.cast.python.jython.test/.launchers/TestCalls.launch
diff --git a/com.ibm.wala.cast.python.jython.test/.launchers/TestTensorflowModel.launch b/com.ibm.wala.cast.python.jython.test/.launchers/TestTensorflowModel.launch
diff --git a/com.ibm.wala.cast.python.jython.test/pom.xml b/com.ibm.wala.cast.python.jython.test/pom.xml
@@ -1,4 +1,4 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="https://maven.apache.org/POM/4.0.0" xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>com.ibm.wala</groupId>
@@ -26,27 +26,27 @@
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.util</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.shrike</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.core</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.cast</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.cast.java</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>

diff --git a/com.ibm.wala.cast.python.jython/data/annots1.py b/com.ibm.wala.cast.python.jython/data/annots1.py
@@ -0,0 +1,3 @@
+def greet(name: str, age: int) -> str:
+    print('Hello {0}, you are {1} years old'.format(name, age))
+
diff --git a/com.ibm.wala.cast.python.jython/data/except1.py b/com.ibm.wala.cast.python.jython/data/except1.py
@@ -0,0 +1,14 @@
+import sys
+
+try:
+    f = open('myfile.txt')
+    s = f.readline()
+    i = int(s.strip())
+except IOError as e:
+    print "I/O error:"
+    print "{0}: {1}".format(e.errno, e.strerror)
+except ValueError:
+    print "Could not convert data to an integer."
+except:
+    print "Unexpected error:", sys.exc_info()[0]
+    raise
diff --git a/com.ibm.wala.cast.python.jython/pom.xml b/com.ibm.wala.cast.python.jython/pom.xml
@@ -1,4 +1,4 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="https://maven.apache.org/POM/4.0.0" xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>com.ibm.wala</groupId>
@@ -11,27 +11,27 @@
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.util</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.shrike</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.core</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.cast</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.cast.java</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
@@ -40,8 +40,8 @@
     </dependency>
     <dependency>
       <groupId>org.python</groupId>
-      <artifactId>jython</artifactId>
-      <version>2.7.2a1+</version>
+      <artifactId>jython-slim</artifactId>
+      <version>2.7.3a1-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>commons-cli</groupId>

diff --git a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/loader/Python2Loader.java
@@ -11,6 +11,7 @@
 package com.ibm.wala.cast.python.loader;
 
 import java.io.IOException;
+import java.util.List;
 
 import org.python.core.PyObject;
 
@@ -29,6 +30,7 @@
 import com.ibm.wala.cast.tree.rewrite.PatternBasedRewriter;
 import com.ibm.wala.cast.util.CAstPattern.Segments;
 import com.ibm.wala.classLoader.IClassLoader;
+import com.ibm.wala.classLoader.Module;
 import com.ibm.wala.classLoader.ModuleEntry;
 import com.ibm.wala.classLoader.SourceModule;
 import com.ibm.wala.ipa.cha.IClassHierarchy;
@@ -43,8 +45,8 @@ public Python2Loader(IClassHierarchy cha) {
 	}
 
 	@Override
-	protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M) throws IOException {
-		RewritingTranslatorToCAst x = new RewritingTranslatorToCAst(M, new PythonModuleParser((SourceModule)M, typeDictionary) {
+	protected TranslatorToCAst getTranslatorToCAst(CAst ast, ModuleEntry M, List<Module> allModules) throws IOException {
+		RewritingTranslatorToCAst x = new RewritingTranslatorToCAst(M, new PythonModuleParser((SourceModule)M, typeDictionary, allModules) {
 			@Override
 			public CAstEntity translateToCAst() throws Error, IOException {
 				CAstEntity ce =  super.translateToCAst();

diff --git a/...bm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java b/...bm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonModuleParser.java
@@ -14,13 +14,16 @@
 import java.io.InputStreamReader;
 import java.io.Reader;
 import java.net.URL;
+import java.util.Collections;
+import java.util.List;
 
 import org.antlr.runtime.ANTLRInputStream;
 import org.antlr.runtime.CharStream;
 
 import com.ibm.wala.cast.tree.CAstEntity;
 import com.ibm.wala.cast.tree.impl.CAstTypeDictionaryImpl;
 import com.ibm.wala.cast.util.CAstPrinter;
+import com.ibm.wala.classLoader.Module;
 import com.ibm.wala.classLoader.ModuleEntry;
 import com.ibm.wala.classLoader.SourceModule;
 import com.ibm.wala.classLoader.SourceURLModule;
@@ -38,7 +41,7 @@ protected WalaPythonParser makeParser() throws IOException {
 		return new WalaPythonParser(file, fileName.getName(), "UTF-8");
 	}
 
-	public PythonModuleParser(SourceModule fileName, CAstTypeDictionaryImpl<String> types) {
+	public PythonModuleParser(SourceModule fileName, CAstTypeDictionaryImpl<String> types, List<Module> allModules) {
 		super(types);
 		this.fileName = fileName;
 	}
@@ -50,7 +53,7 @@ protected String scriptName() {
 
 	public static void main(String[] args) throws Exception {
 		URL url = new URL(args[0]);
-		PythonParser<ModuleEntry> p = new PythonModuleParser(new SourceURLModule(url), new CAstTypeDictionaryImpl<String>());
+		PythonParser<ModuleEntry> p = new PythonModuleParser(new SourceURLModule(url), new CAstTypeDictionaryImpl<String>(), Collections.singletonList(new SourceURLModule(url)));
 		CAstEntity script = p.translateToCAst();
 		System.err.println(script);
 		System.err.println(CAstPrinter.print(script));

diff --git a/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python.jython/source/com/ibm/wala/cast/python/parser/PythonParser.java
@@ -118,7 +118,7 @@
 import com.ibm.wala.util.collections.HashMapFactory;
 import com.ibm.wala.util.collections.HashSetFactory;
 import com.ibm.wala.util.collections.ReverseIterator;
-import com.ibm.wala.util.warnings.Warning;
+import com.ibm.wala.core.util.warnings.Warning;
 
 abstract public class PythonParser<T> extends AbstractParser<T> implements TranslatorToCAst {
 
@@ -232,7 +232,7 @@ private Position makePosition(PythonTree p) {
 			int last_col;
 			int last_line = p.getLineno() + lines.length - 1;
 			if ("".equals(s) || lines.length <= 1) {
-				last_col = p.getCharPositionInLine() + (p.getCharStopIndex() - p.getCharStartIndex());
+				last_col = p.getCol_offset() + (p.getCharStopIndex() - p.getCharStartIndex());
 			} else {
 				assert (lines.length > 1);
 				last_col = lines[lines.length-1].length();
@@ -262,7 +262,7 @@ public int getFirstLine() {
 
 				@Override
 				public int getFirstCol() {
-					return p.getCharPositionInLine();
+					return p.getCol_offset();
 				}
 
 				@Override
@@ -1092,7 +1092,7 @@ public CAstNode visitImport(Import arg0) throws Exception {
 			int i = 0;
 			CAstNode[] elts = new CAstNode[ arg0.getInternalNames().size() ];
 			for(alias n : arg0.getInternalNames()) {
-				CAstNode obj = importAst(n.getInternalNameNodes());
+				CAstNode obj = importAst(arg0, n.getInternalNameNodes());
 				elts[i++] = notePosition(Ast.makeNode(CAstNode.DECL_STMT,
 					Ast.makeConstant(new CAstSymbolImpl(name(n), PythonCAstToIRTranslator.Any)),
 					obj != null?
@@ -1109,7 +1109,7 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception {
 
 			elts[0] = Ast.makeNode(CAstNode.DECL_STMT,
 				Ast.makeConstant(new CAstSymbolImpl(tree, PythonCAstToIRTranslator.Any)),
-				importAst(arg0.getInternalModuleNames()));					
+				importAst(arg0, arg0.getInternalModuleNames()));					
 
 			int i = 1;
 			for(alias n : arg0.getInternalNames()) {
@@ -1123,10 +1123,12 @@ public CAstNode visitImportFrom(ImportFrom arg0) throws Exception {
 			return Ast.makeNode(CAstNode.BLOCK_STMT, elts);
 		}
 
-		private CAstNode importAst(java.util.List<Name> names ) {
+		private final boolean wholeStatement = true;
+
+		private <R extends stmt> CAstNode importAst(R importNode, java.util.List<Name> names ) {
 			CAstNode importAst = notePosition(Ast.makeNode(CAstNode.PRIMITIVE, 
 				Ast.makeConstant("import"), 
-				Ast.makeConstant(names.get(0).getInternalId())), names.get(0));
+				Ast.makeConstant(names.get(0).getInternalId())), wholeStatement? importNode: names.get(0));
 			for(int i = 1; i < names.size(); i++) {
 				importAst = notePosition(Ast.makeNode(CAstNode.OBJECT_REF,
 					importAst,

diff --git a/com.ibm.wala.cast.python.jython3.test/.classpath b/com.ibm.wala.cast.python.jython3.test/.classpath
@@ -17,19 +17,18 @@
 	</classpathentry>
 	<classpathentry kind="src" output="target/test-classes" path="test-source">
 		<attributes>
+			<attribute name="test" value="true"/>
 			<attribute name="gradle_scope" value="test"/>
 			<attribute name="gradle_used_by_scope" value="test"/>
 			<attribute name="optional" value="true"/>
 			<attribute name="maven.pomderived" value="true"/>
-			<attribute name="test" value="true"/>
 		</attributes>
 	</classpathentry>
 	<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>
 		</attributes>
 	</classpathentry>
-	<classpathentry kind="con" path="org.eclipse.buildship.core.gradleclasspathcontainer"/>
 	<classpathentry kind="con" path="org.eclipse.m2e.MAVEN2_CLASSPATH_CONTAINER">
 		<attributes>
 			<attribute name="maven.pomderived" value="true"/>

diff --git a/com.ibm.wala.cast.python.jython3.test/.launchers/TestCalls.launch b/com.ibm.wala.cast.python.jython3.test/.launchers/TestCalls.launch
diff --git a/com.ibm.wala.cast.python.jython3.test/.launchers/TestTensorflowModel.launch b/com.ibm.wala.cast.python.jython3.test/.launchers/TestTensorflowModel.launch
diff --git a/com.ibm.wala.cast.python.jython3.test/pom.xml b/com.ibm.wala.cast.python.jython3.test/pom.xml
@@ -1,4 +1,4 @@
-<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+<project xmlns="https://maven.apache.org/POM/4.0.0" xmlns:xsi="https://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="https://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
   <modelVersion>4.0.0</modelVersion>
   <parent>
     <groupId>com.ibm.wala</groupId>
@@ -26,27 +26,27 @@
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.util</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.shrike</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.core</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.cast</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>com.ibm.wala</groupId>
       <artifactId>com.ibm.wala.cast.java</artifactId>
-      <version>1.5.6-SNAPSHOT</version>
+      <version>1.5.8-SNAPSHOT</version>
     </dependency>
     <dependency>
       <groupId>junit</groupId>