From 106d7086211aae33bb272f0cd2f8987fdf713d01 Mon Sep 17 00:00:00 2001 From: Raffi Khatchadourian Date: Thu, 25 Apr 2024 15:46:11 -0400 Subject: [PATCH] Support static methods (#186) Based on static method support described in https://github.com/Anemone95/wala-python?tab=readme-ov-file#new-features. --- .../wala/cast/python/parser/PythonParser.java | 56 +++++- .../python/ml/test/TestTensorflow2Model.java | 186 ++++++++++++++++++ .../data/tf2_test_static_method.py | 11 ++ .../data/tf2_test_static_method10.py | 12 ++ .../data/tf2_test_static_method11.py | 17 ++ .../data/tf2_test_static_method12.py | 16 ++ .../data/tf2_test_static_method2.py | 12 ++ .../data/tf2_test_static_method3.py | 11 ++ .../data/tf2_test_static_method4.py | 10 + .../data/tf2_test_static_method5.py | 12 ++ .../data/tf2_test_static_method6.py | 11 ++ .../data/tf2_test_static_method7.py | 11 ++ .../data/tf2_test_static_method8.py | 10 + .../data/tf2_test_static_method9.py | 13 ++ .../PythonTrampolineTargetSelector.java | 50 +++-- .../wala/cast/python/loader/PythonLoader.java | 36 +++- .../wala/cast/python/types/PythonTypes.java | 27 +++ .../com/ibm/wala/cast/python/util/Util.java | 36 ++++ 18 files changed, 512 insertions(+), 25 deletions(-) create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method10.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method11.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method12.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method2.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method3.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method4.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method5.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method6.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method7.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method8.py create mode 100644 com.ibm.wala.cast.python.test/data/tf2_test_static_method9.py diff --git a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java index 783ea4aa8..4ae07fc6f 100644 --- a/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java +++ b/com.ibm.wala.cast.python.jython3/source/com/ibm/wala/cast/python/parser/PythonParser.java @@ -10,6 +10,10 @@ *****************************************************************************/ package com.ibm.wala.cast.python.parser; +import static com.ibm.wala.cast.python.util.Util.DYNAMIC_ANNOTATION_KEY; +import static com.ibm.wala.cast.python.util.Util.STATIC_METHOD_ANNOTATION_NAME; +import static com.ibm.wala.cast.python.util.Util.getNameStream; + import com.ibm.wala.cast.ir.translator.AbstractClassEntity; import com.ibm.wala.cast.ir.translator.AbstractCodeEntity; import com.ibm.wala.cast.ir.translator.AbstractFieldEntity; @@ -19,6 +23,7 @@ import com.ibm.wala.cast.python.loader.DynamicAnnotatableEntity; import com.ibm.wala.cast.python.types.PythonTypes; import com.ibm.wala.cast.tree.CAst; +import com.ibm.wala.cast.tree.CAstAnnotation; import com.ibm.wala.cast.tree.CAstEntity; import com.ibm.wala.cast.tree.CAstNode; import com.ibm.wala.cast.tree.CAstQualifier; @@ -49,6 +54,7 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; +import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.Map; @@ -1155,6 +1161,35 @@ public String toString() { } ; + Collection annotations = new ArrayList<>(); + + for (CAstNode node : dynamicAnnotations) { + CAstAnnotation cAstAnnotation = + new CAstAnnotation() { + @Override + public CAstType getType() { + return PythonTypes.CAST_DYNAMIC_ANNOTATION; + } + + @Override + public Map getArguments() { + Map map = new HashMap<>(); + map.put(DYNAMIC_ANNOTATION_KEY, node); + return map; + } + + @Override + public String toString() { + return this.getArguments().getOrDefault(DYNAMIC_ANNOTATION_KEY, this).toString(); + } + }; + + annotations.add(cAstAnnotation); + } + + boolean staticMethod = + getNameStream(annotations).anyMatch(s -> s.equals(STATIC_METHOD_ANNOTATION_NAME)); + CAstType functionType; boolean isMethod = context.entity().getKind() == CAstEntity.TYPE_ENTITY @@ -1170,7 +1205,7 @@ public CAstType getDeclaringType() { @Override public boolean isStatic() { - return false; + return staticMethod; } } ; @@ -1199,14 +1234,25 @@ class PythonCodeEntity extends AbstractCodeEntity implements PythonGlobalsEntity, DynamicAnnotatableEntity { private final java.util.Set downwardGlobals; + private final Collection annotations; + @Override public Iterable dynamicAnnotations() { return dynamicAnnotations; } - protected PythonCodeEntity(CAstType type, java.util.Set downwardGlobals) { + protected PythonCodeEntity( + CAstType type, + java.util.Set downwardGlobals, + Collection annotations) { super(type); this.downwardGlobals = downwardGlobals; + this.annotations = annotations; + } + + @Override + public Collection getAnnotations() { + return this.annotations; } @Override @@ -1217,8 +1263,10 @@ public int getKind() { @Override public CAstNode getAST() { if (function instanceof FunctionDef) { - if (isMethod) { + // Only add object metadata for non-static methods. + if (isMethod && !staticMethod) { CAst Ast = PythonParser.this.Ast; + CAstNode[] newNodes = new CAstNode[nodes.length + 2]; System.arraycopy(nodes, 0, newNodes, 2, nodes.length); @@ -1306,7 +1354,7 @@ public java.util.Set downwardGlobals() { java.util.Set downwardGlobals = HashSetFactory.make(); - PythonCodeEntity fun = new PythonCodeEntity(functionType, downwardGlobals); + PythonCodeEntity fun = new PythonCodeEntity(functionType, downwardGlobals, annotations); PythonParser.FunctionContext child = new PythonParser.FunctionContext(context, fun, downwardGlobals, function); diff --git a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflow2Model.java b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflow2Model.java index 2e3f5cddd..ed9c5a2ff 100644 --- a/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflow2Model.java +++ b/com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflow2Model.java @@ -3314,6 +3314,192 @@ public void testModule53() expectedTensorParameterValueNumbers); } + @Test + public void testStaticMethod() throws ClassHierarchyException, CancelException, IOException { + test("tf2_test_static_method.py", "MyClass.the_static_method", 1, 1, 2); + } + + @Test + public void testStaticMethod2() throws ClassHierarchyException, CancelException, IOException { + test("tf2_test_static_method2.py", "MyClass.the_static_method", 1, 1, 2); + } + + @Test + public void testStaticMethod3() throws ClassHierarchyException, CancelException, IOException { + test("tf2_test_static_method3.py", "MyClass.the_static_method", 1, 1, 2); + } + + @Test + public void testStaticMethod4() throws ClassHierarchyException, CancelException, IOException { + test("tf2_test_static_method4.py", "MyClass.the_static_method", 1, 1, 2); + } + + @Test + public void testStaticMethod5() throws ClassHierarchyException, CancelException, IOException { + int expectNumberofTensorParameters; + int expectedNumberOfTensorVariables; + int[] expectedTensorParameterValueNumbers; + + // Static methods are only supported for Jython3. + if (usesJython3Testing()) { + expectNumberofTensorParameters = 1; + expectedNumberOfTensorVariables = 1; + expectedTensorParameterValueNumbers = new int[] {2}; + } else { + // NOTE: Remove this case once https://github.com/wala/ML/issues/147 is fixed. + expectNumberofTensorParameters = 0; + expectedNumberOfTensorVariables = 0; + expectedTensorParameterValueNumbers = new int[] {}; + } + + test( + "tf2_test_static_method5.py", + "MyClass.the_static_method", + expectNumberofTensorParameters, + expectedNumberOfTensorVariables, + expectedTensorParameterValueNumbers); + } + + @Test + public void testStaticMethod6() throws ClassHierarchyException, CancelException, IOException { + int expectNumberofTensorParameters; + int expectedNumberOfTensorVariables; + int[] expectedTensorParameterValueNumbers; + + // Static methods are only supported for Jython3. + if (usesJython3Testing()) { + expectNumberofTensorParameters = 1; + expectedNumberOfTensorVariables = 1; + expectedTensorParameterValueNumbers = new int[] {2}; + } else { + // NOTE: Remove this case once https://github.com/wala/ML/issues/147 is fixed. + expectNumberofTensorParameters = 0; + expectedNumberOfTensorVariables = 0; + expectedTensorParameterValueNumbers = new int[] {}; + } + + test( + "tf2_test_static_method6.py", + "MyClass.the_static_method", + expectNumberofTensorParameters, + expectedNumberOfTensorVariables, + expectedTensorParameterValueNumbers); + } + + @Test + public void testStaticMethod7() throws ClassHierarchyException, CancelException, IOException { + test("tf2_test_static_method7.py", "MyClass.the_static_method", 1, 1, 3); + } + + @Test + public void testStaticMethod8() throws ClassHierarchyException, CancelException, IOException { + test("tf2_test_static_method8.py", "MyClass.the_static_method", 1, 1, 3); + } + + @Test + public void testStaticMethod9() throws ClassHierarchyException, CancelException, IOException { + int expectNumberofTensorParameters; + int expectedNumberOfTensorVariables; + int[] expectedTensorParameterValueNumbers; + + // Static methods are only supported for Jython3. + if (usesJython3Testing()) { + expectNumberofTensorParameters = 2; + expectedNumberOfTensorVariables = 2; + expectedTensorParameterValueNumbers = new int[] {2, 3}; + } else { + // NOTE: Remove this case once https://github.com/wala/ML/issues/147 is fixed. + expectNumberofTensorParameters = 1; + expectedNumberOfTensorVariables = 1; + expectedTensorParameterValueNumbers = new int[] {3}; + } + + test( + "tf2_test_static_method9.py", + "MyClass.the_static_method", + expectNumberofTensorParameters, + expectedNumberOfTensorVariables, + expectedTensorParameterValueNumbers); + } + + @Test + public void testStaticMethod10() throws ClassHierarchyException, CancelException, IOException { + int expectNumberofTensorParameters; + int expectedNumberOfTensorVariables; + int[] expectedTensorParameterValueNumbers; + + // Static methods are only supported for Jython3. + if (usesJython3Testing()) { + expectNumberofTensorParameters = 2; + expectedNumberOfTensorVariables = 2; + expectedTensorParameterValueNumbers = new int[] {2, 3}; + } else { + // NOTE: Remove this case once https://github.com/wala/ML/issues/147 is fixed. + expectNumberofTensorParameters = 1; + expectedNumberOfTensorVariables = 1; + expectedTensorParameterValueNumbers = new int[] {3}; + } + + test( + "tf2_test_static_method10.py", + "MyClass.the_static_method", + expectNumberofTensorParameters, + expectedNumberOfTensorVariables, + expectedTensorParameterValueNumbers); + } + + @Test + public void testStaticMethod11() throws ClassHierarchyException, CancelException, IOException { + int expectNumberofTensorParameters; + int expectedNumberOfTensorVariables; + int[] expectedTensorParameterValueNumbers; + + // Static methods are only supported for Jython3. + if (usesJython3Testing()) { + expectNumberofTensorParameters = 1; + expectedNumberOfTensorVariables = 1; + expectedTensorParameterValueNumbers = new int[] {2}; + } else { + // NOTE: Remove this case once https://github.com/wala/ML/issues/147 is fixed. + expectNumberofTensorParameters = 0; + expectedNumberOfTensorVariables = 0; + expectedTensorParameterValueNumbers = new int[] {}; + } + + test( + "tf2_test_static_method11.py", + "f", + expectNumberofTensorParameters, + expectedNumberOfTensorVariables, + expectedTensorParameterValueNumbers); + } + + @Test + public void testStaticMethod12() throws ClassHierarchyException, CancelException, IOException { + int expectNumberofTensorParameters; + int expectedNumberOfTensorVariables; + int[] expectedTensorParameterValueNumbers; + + // Static methods are only supported for Jython3. + if (usesJython3Testing()) { + expectNumberofTensorParameters = 1; + expectedNumberOfTensorVariables = 1; + expectedTensorParameterValueNumbers = new int[] {2}; + } else { + // NOTE: Remove this case once https://github.com/wala/ML/issues/147 is fixed. + expectNumberofTensorParameters = 0; + expectedNumberOfTensorVariables = 0; + expectedTensorParameterValueNumbers = new int[] {}; + } + + test( + "tf2_test_static_method12.py", + "f", + expectNumberofTensorParameters, + expectedNumberOfTensorVariables, + expectedTensorParameterValueNumbers); + } + private void test( String filename, String functionName, diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method.py new file mode 100644 index 000000000..5012d8fc3 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method.py @@ -0,0 +1,11 @@ +import tensorflow as tf + + +class MyClass: + + @staticmethod + def the_static_method(x): + assert isinstance(x, tf.Tensor) + + +MyClass.the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method10.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method10.py new file mode 100644 index 000000000..815e68811 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method10.py @@ -0,0 +1,12 @@ +import tensorflow as tf + + +class MyClass: + + @staticmethod + def the_static_method(x, y): + assert isinstance(x, tf.Tensor) + assert isinstance(y, tf.Tensor) + + +MyClass().the_static_method(tf.constant(1), tf.constant(2)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method11.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method11.py new file mode 100644 index 000000000..87e956aa2 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method11.py @@ -0,0 +1,17 @@ +import tensorflow as tf + + +def f(x): + assert isinstance(x, tf.Tensor) + + +class MyClass: + + @staticmethod + @tf.function + def the_static_method(x): + assert isinstance(x, tf.Tensor) + f(x) + + +MyClass().the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method12.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method12.py new file mode 100644 index 000000000..90b5e507f --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method12.py @@ -0,0 +1,16 @@ +import tensorflow as tf + + +def f(x): + assert isinstance(x, tf.Tensor) + + +class MyClass: + + @staticmethod + def the_static_method(x): + assert isinstance(x, tf.Tensor) + f(x) + + +MyClass().the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method2.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method2.py new file mode 100644 index 000000000..8fb9b8b4e --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method2.py @@ -0,0 +1,12 @@ +import tensorflow as tf + + +class MyClass: + + @staticmethod + @tf.function + def the_static_method(x): + assert isinstance(x, tf.Tensor) + + +MyClass.the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method3.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method3.py new file mode 100644 index 000000000..b4b4344de --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method3.py @@ -0,0 +1,11 @@ +import tensorflow as tf + + +class MyClass: + + @tf.function + def the_static_method(x): + assert isinstance(x, tf.Tensor) + + +MyClass.the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method4.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method4.py new file mode 100644 index 000000000..330d1849c --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method4.py @@ -0,0 +1,10 @@ +import tensorflow as tf + + +class MyClass: + + def the_static_method(x): + assert isinstance(x, tf.Tensor) + + +MyClass.the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method5.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method5.py new file mode 100644 index 000000000..1e85fa573 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method5.py @@ -0,0 +1,12 @@ +import tensorflow as tf + + +class MyClass: + + @staticmethod + @tf.function + def the_static_method(x): + assert isinstance(x, tf.Tensor) + + +MyClass().the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method6.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method6.py new file mode 100644 index 000000000..ca95b5b7d --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method6.py @@ -0,0 +1,11 @@ +import tensorflow as tf + + +class MyClass: + + @staticmethod + def the_static_method(x): + assert isinstance(x, tf.Tensor) + + +MyClass().the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method7.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method7.py new file mode 100644 index 000000000..4d4d03ce7 --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method7.py @@ -0,0 +1,11 @@ +import tensorflow as tf + + +class MyClass: + + @tf.function + def the_static_method(self, x): + assert isinstance(x, tf.Tensor) + + +MyClass().the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method8.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method8.py new file mode 100644 index 000000000..9c867880a --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method8.py @@ -0,0 +1,10 @@ +import tensorflow as tf + + +class MyClass: + + def the_static_method(self, x): + assert isinstance(x, tf.Tensor) + + +MyClass().the_static_method(tf.constant(1)) diff --git a/com.ibm.wala.cast.python.test/data/tf2_test_static_method9.py b/com.ibm.wala.cast.python.test/data/tf2_test_static_method9.py new file mode 100644 index 000000000..7e804594c --- /dev/null +++ b/com.ibm.wala.cast.python.test/data/tf2_test_static_method9.py @@ -0,0 +1,13 @@ +import tensorflow as tf + + +class MyClass: + + @staticmethod + @tf.function + def the_static_method(x, y): + assert isinstance(x, tf.Tensor) + assert isinstance(y, tf.Tensor) + + +MyClass().the_static_method(tf.constant(1), tf.constant(2)) diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java index b5982e721..5ea98489a 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/ipa/callgraph/PythonTrampolineTargetSelector.java @@ -10,6 +10,9 @@ *****************************************************************************/ package com.ibm.wala.cast.python.ipa.callgraph; +import static com.ibm.wala.cast.python.types.PythonTypes.STATIC_METHOD; +import static com.ibm.wala.types.annotations.Annotation.make; + import com.ibm.wala.cast.ipa.callgraph.ScopeMappingInstanceKeys.ScopeMappingInstanceKey; import com.ibm.wala.cast.loader.DynamicCallSiteReference; import com.ibm.wala.cast.python.client.PythonAnalysisEngine; @@ -83,6 +86,9 @@ public PythonTrampolineTargetSelector( @Override public IMethod getCalleeTarget(CGNode caller, CallSiteReference site, IClass receiver) { if (receiver != null) { + logger.fine("Getting callee target for receiver: " + receiver); + logger.fine("Calling method name is: " + caller.getMethod().getName()); + IClassHierarchy cha = receiver.getClassHierarchy(); final boolean callable = receiver.getReference().equals(PythonTypes.object); @@ -110,6 +116,14 @@ public IMethod getCalleeTarget(CGNode caller, CallSiteReference site, IClass rec AstMethodReference.fnDesc); PythonSummary x = new PythonSummary(tr, call.getNumberOfTotalParameters()); IClass filter = ((PythonInstanceMethodTrampoline) receiver).getRealClass(); + + // Are we calling a static method? + boolean staticMethodReceiver = filter.getAnnotations().contains(make(STATIC_METHOD)); + logger.fine( + staticMethodReceiver + ? "Found static method receiver: " + filter + : "Method is not static: " + filter); + int v = call.getNumberOfTotalParameters() + 1; x.addStatement( @@ -129,23 +143,33 @@ public IMethod getCalleeTarget(CGNode caller, CallSiteReference site, IClass rec PythonLanguage.Python.instructionFactory() .CheckCastInstruction(1, v0, v, filter.getReference(), true)); - int v1 = v + 2; + int v1; - x.addStatement( - PythonLanguage.Python.instructionFactory() - .GetInstruction( - 1, - v1, - 1, - FieldReference.findOrCreate( - PythonTypes.Root, - Atom.findOrCreateUnicodeAtom("$self"), - PythonTypes.Root))); + // only add self if the receiver isn't static. + if (!staticMethodReceiver) { + v1 = v + 2; + + x.addStatement( + PythonLanguage.Python.instructionFactory() + .GetInstruction( + 1, + v1, + 1, + FieldReference.findOrCreate( + PythonTypes.Root, + Atom.findOrCreateUnicodeAtom("$self"), + PythonTypes.Root))); + } else v1 = v + 1; int i = 0; - int[] params = new int[Math.max(2, call.getNumberOfPositionalParameters() + 1)]; + int paramSize = + Math.max( + staticMethodReceiver ? 1 : 2, + call.getNumberOfPositionalParameters() + (staticMethodReceiver ? 0 : 1)); + int[] params = new int[paramSize]; params[i++] = v0; - params[i++] = v1; + + if (!staticMethodReceiver) params[i++] = v1; for (int j = 1; j < call.getNumberOfPositionalParameters(); j++) params[i++] = j + 1; diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java index fe9c0a287..c5a0c0ebf 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/loader/PythonLoader.java @@ -1,5 +1,9 @@ package com.ibm.wala.cast.python.loader; +import static com.ibm.wala.cast.python.types.PythonTypes.pythonLoader; +import static com.ibm.wala.cast.python.util.Util.getNameStream; +import static java.util.stream.Collectors.toList; + import com.ibm.wala.cast.ir.translator.AstTranslator.AstLexicalInformation; import com.ibm.wala.cast.ir.translator.AstTranslator.WalkContext; import com.ibm.wala.cast.ir.translator.TranslatorToIR; @@ -60,6 +64,8 @@ public void init(List modules) { public class DynamicMethodBody extends DynamicCodeBody { private final IClass container; + private final Collection annotations; + public DynamicMethodBody( TypeReference codeName, TypeReference parent, @@ -70,11 +76,26 @@ public DynamicMethodBody( IClass container) { super(codeName, parent, loader, sourcePosition, entity, context); this.container = container; + + // fill in the decorators. + // FIXME: Process annotations with parameters. + this.annotations = + getNameStream(entity.getAnnotations()) + .map(s -> "L" + s) + .map(TypeName::findOrCreate) + .map(tn -> TypeReference.findOrCreate(pythonLoader, tn)) + .map(Annotation::make) + .collect(toList()); } public IClass getContainer() { return container; } + + @Override + public Collection getAnnotations() { + return this.annotations; + } } public class PythonClass extends CoreClass { @@ -290,14 +311,13 @@ public IClass defineMethodType( assert types.containsKey(typeName); - if (entity.getArgumentCount() > 0 && "self".equals(entity.getArgumentNames()[1])) { - MethodReference me = - MethodReference.findOrCreate( - fun.getReference(), - Atom.findOrCreateUnicodeAtom(entity.getType().getName()), - AstMethodReference.fnDesc); - self.methodTypes.add(me); - } + // Includes static methods. + MethodReference me = + MethodReference.findOrCreate( + fun.getReference(), + Atom.findOrCreateUnicodeAtom(entity.getType().getName()), + AstMethodReference.fnDesc); + self.methodTypes.add(me); return fun; } diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java index 6a15512aa..1c2a89f6e 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/types/PythonTypes.java @@ -10,11 +10,16 @@ *****************************************************************************/ package com.ibm.wala.cast.python.types; +import static com.ibm.wala.cast.python.util.Util.STATIC_METHOD_ANNOTATION_NAME; + +import com.ibm.wala.cast.tree.CAstType; import com.ibm.wala.cast.types.AstTypeReference; import com.ibm.wala.core.util.strings.Atom; import com.ibm.wala.types.ClassLoaderReference; import com.ibm.wala.types.TypeName; import com.ibm.wala.types.TypeReference; +import java.util.Collection; +import java.util.HashSet; public class PythonTypes extends AstTypeReference { @@ -22,6 +27,9 @@ public class PythonTypes extends AstTypeReference { public static final String pythonLoaderNameStr = "PythonLoader"; + /** The name of the type used for CAst dynamic annotations (decorators). */ + private static final String DYNAMIC_ANNOTATION_TYPE_NAME = "DYNAMIC_ANNOTATION"; + public static final Atom pythonName = Atom.findOrCreateUnicodeAtom(pythonNameStr); public static final Atom pythonLoaderName = Atom.findOrCreateUnicodeAtom(pythonLoaderNameStr); @@ -76,4 +84,23 @@ public class PythonTypes extends AstTypeReference { /** https://docs.python.org/3/library/stdtypes.html#typeiter. */ public static final TypeReference iterator = TypeReference.findOrCreate(pythonLoader, TypeName.findOrCreate("Literator")); + + /** https://docs.python.org/3/library/functions.html#staticmethod. */ + public static final TypeReference STATIC_METHOD = + TypeReference.findOrCreate( + pythonLoader, TypeName.findOrCreate("L" + STATIC_METHOD_ANNOTATION_NAME)); + + /** A {@link CAstType} representing a dynamic annotation (decorator). */ + public static final CAstType CAST_DYNAMIC_ANNOTATION = + new CAstType() { + @Override + public String getName() { + return DYNAMIC_ANNOTATION_TYPE_NAME; + } + + @Override + public Collection getSupertypes() { + return new HashSet<>(); + } + }; } diff --git a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/util/Util.java b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/util/Util.java index 06a970345..4ad4f151e 100644 --- a/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/util/Util.java +++ b/com.ibm.wala.cast.python/source/com/ibm/wala/cast/python/util/Util.java @@ -1,21 +1,33 @@ package com.ibm.wala.cast.python.util; import static com.google.common.collect.Iterables.concat; +import static com.ibm.wala.cast.python.types.PythonTypes.CAST_DYNAMIC_ANNOTATION; import static java.util.Collections.emptyList; import static java.util.stream.Collectors.toList; import com.ibm.wala.cast.python.ipa.callgraph.PytestEntrypointBuilder; +import com.ibm.wala.cast.tree.CAstAnnotation; +import com.ibm.wala.cast.tree.CAstNode; import com.ibm.wala.ipa.callgraph.Entrypoint; import com.ibm.wala.ipa.callgraph.propagation.PropagationCallGraphBuilder; import java.io.File; import java.util.Arrays; +import java.util.Collection; import java.util.List; +import java.util.Objects; import java.util.logging.Logger; +import java.util.stream.Stream; public class Util { private static final Logger LOGGER = Logger.getLogger(Util.class.getName()); + /** Key used to map annotations (decorators) to names. */ + public static final String DYNAMIC_ANNOTATION_KEY = "dynamicAnnotation"; + + /** Name of the annotation (decorator) that marks methods as static. */ + public static final String STATIC_METHOD_ANNOTATION_NAME = "staticmethod"; + /** * Add Pytest entrypoints to the given {@link PropagationCallGraphBuilder}. * @@ -50,5 +62,29 @@ public static List getPathFiles(String pathSequence) { return Arrays.asList(pathSequence.split(":")).stream().map(File::new).collect(toList()); } + /** + * Returns a {@link Stream} of annotation (decorator) names as {@link String}s from the given + * {@link Collection} of {@link CAstAnnotation}s. + * + * @param annotations A {@link Collection} of {@link CAstAnnotation} for which to stream + * annotation (decorator) names. + * @return A {@link Stream} of names as {@link String}s corresponding to the given annotations + * (decorators). + */ + public static Stream getNameStream(Collection annotations) { + if (annotations == null) return Stream.empty(); + + return annotations.stream() + .filter(a -> a.getType().equals(CAST_DYNAMIC_ANNOTATION)) + .map(a -> a.getArguments().get(DYNAMIC_ANNOTATION_KEY)) + .filter(Objects::nonNull) + .map(CAstNode.class::cast) + .map(n -> n.getChild(0)) + .map(n -> n.getChild(0)) + .map(CAstNode::getValue) + .filter(v -> v instanceof String) + .map(String.class::cast); + } + private Util() {} }