Skip to content

Commit

Permalink
move pandas model out of ML
Browse files Browse the repository at this point in the history 
updates to LSP version
fixes
  • Loading branch information
@juliandolby
juliandolby committed Oct 3, 2018
1 parent 722ea0e commit c9ed3d2
Show file tree
Hide file tree
Showing 22 changed files with 551 additions and 303 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ install:
- mvn clean install -DskipTests -B
- popd
script:
- mvn clean install -B -q
- mvn clean install -B
sudo: false
deploy:
provider: pages
Expand Down
2 changes: 1 addition & 1 deletion com.ibm.wala.cast.python.ml.test/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@
<dependency>
<groupId>org.eclipse.lsp4j</groupId>
<artifactId>org.eclipse.lsp4j</artifactId>
<version>0.6.0-SNAPSHOT</version>
<version>0.5.0</version>
</dependency>
<dependency>
<groupId>junit</groupId>
Expand Down
97 changes: 72 additions & 25 deletions com.ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/ServerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@

import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.net.Socket;
import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
Expand All @@ -14,13 +14,17 @@
import java.util.concurrent.ExecutionException;

import org.eclipse.lsp4j.CodeLens;
import org.eclipse.lsp4j.Command;
import org.eclipse.lsp4j.Diagnostic;
import org.eclipse.lsp4j.PublishDiagnosticsParams;
import org.eclipse.lsp4j.Range;
import org.eclipse.lsp4j.SymbolInformation;
import org.eclipse.lsp4j.TextEdit;
import org.junit.Test;

import com.ibm.wala.cast.lsp.WALAServer;
import com.ibm.wala.cast.python.ml.driver.ClientDriver;
import com.ibm.wala.cast.python.ml.driver.PythonDriver;
import com.ibm.wala.core.tests.util.WalaTestCase;
import com.ibm.wala.ipa.cha.ClassHierarchyException;
import com.ibm.wala.util.CancelException;
Expand All @@ -31,57 +35,100 @@ public class ServerTest extends WalaTestCase {

@Test
public void trivialClient() throws IOException, InterruptedException, ExecutionException, ClassHierarchyException, IllegalArgumentException, CancelException, URISyntaxException {
assumeThat("not running on Travis CI", System.getenv("TRAVIS"), nullValue());
//assumeThat("not running on Travis CI", System.getenv("TRAVIS"), nullValue());

String mlFullJar = getClasspathEntry("com.ibm.wala.cast.python.ml-0.0.1");
Process p = Runtime.getRuntime().exec("java -jar " + mlFullJar + " -mode stdio");

String script = "buggy_convolutional_network.py";
WALAServer wala = WALAServer.launchOnServerPort(0, PythonDriver.python, true);

Socket socket = new Socket("localhost", wala.getServerPort());

String script = "buggy_convolutional_network.py";
String fileName = getScript(script);
Set<String> checks = HashSetFactory.make();
ClientDriver.main(new String[] {fileName, "43", "10", "46", "35"}, p.getInputStream(), new PrintStream(p.getOutputStream(), true), (Object s) -> {
ClientDriver.main(new String[] {fileName, "37", "28", "46", "35"}, socket.getInputStream(), socket.getOutputStream(), (Object s) -> {
if (s == null) {
return;
}
if (s.toString().contains("pixel[?][28][28][1]")) {
checks.add("tensor");
//System.err.println(s);
if (s instanceof TextEdit) {
synchronized (checks) {
checks.add(((TextEdit)s).getNewText());
}
} else if (s.toString().contains("pixel[?][28][28][1]")) {
synchronized (checks) {
checks.add("tensor");
}
} else if (s instanceof PublishDiagnosticsParams) {
if (((PublishDiagnosticsParams)s).getDiagnostics().size() > 0) {
checks.add("error");
synchronized (checks) {
checks.add("error");
}
for(Diagnostic d : ((PublishDiagnosticsParams) s).getDiagnostics()) {
Range r = d.getRange();
if (r.getStart().getLine() == 37 &&
r.getStart().getCharacter() == 27 &&
r.getEnd().getLine() == 37 &&
r.getEnd().getCharacter() == 30) {
checks.add("xxx");
synchronized (checks) {
checks.add("xxx");
}
}
}
}
} else if (s instanceof SymbolInformation) {
checks.add(((SymbolInformation)s).getName());
synchronized (checks) {
checks.add(((SymbolInformation)s).getName());
}
} else if (s instanceof CodeLens) {
checks.add(((CodeLens)s).getCommand().getCommand());
synchronized (checks) {
checks.add(((CodeLens)s).getCommand().getCommand());
}
} else if (s instanceof Command) {
synchronized (checks) {
checks.add(((Command)s).getCommand());
}
}
});
System.err.println(checks);
assert (checks.contains("tensor") &&

String stuff = "";
boolean done = false;
while (!done) {
synchronized (checks) {
String now = checks.toString();

if (! now.equals(stuff)) {
stuff = now;
System.err.println(now);
}
}

done = true;
if (! (checks.contains("tensor") &&
checks.contains("error") &&
checks.contains("xxx") &&
checks.contains("TYPES"));
checks.contains("TYPES") &&
checks.contains("FIXES") &&
checks.contains("tf.reshape(xxx, [-1, 28, 28, 1])"))) {
done = false;
}

for(String fun : new String[] {"model_fn", "conv_net" }) {
boolean model_fn = false;
for(String c : checks) {
if (c.endsWith(fun)) {
model_fn = true;
for(String fun : new String[] {"model_fn", "conv_net" }) {
boolean model_fn = false;

synchronized (checks) {
for(String c : checks) {
if (c.endsWith(fun)) {
model_fn = true;
}
}
}

if (! model_fn) {
done = false;
}
}

assert model_fn : "cannot find " + fun + " in assertions";
}

p.destroy();
assert done;
}

private String getScript(String script) throws URISyntaxException, IOException {
Expand Down
16 changes: 2 additions & 14 deletions ...ibm.wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestPandasModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,36 +5,24 @@
import java.util.Set;

import org.junit.Test;

import com.ibm.wala.cast.python.client.PythonAnalysisEngine;
import com.ibm.wala.cast.python.ml.analysis.PandasReadExcelAnalysis;
import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis;
import com.ibm.wala.cast.python.ssa.PythonInstructionVisitor;
import com.ibm.wala.cast.python.ssa.PythonInvokeInstruction;
import com.ibm.wala.cast.python.types.PythonTypes;
import com.ibm.wala.cast.types.AstMethodReference;
import com.ibm.wala.ipa.callgraph.CGNode;
import com.ibm.wala.ipa.callgraph.CallGraph;
import com.ibm.wala.ipa.callgraph.CallGraphBuilder;
import com.ibm.wala.ipa.callgraph.propagation.HeapModel;
import com.ibm.wala.ipa.callgraph.propagation.InstanceKey;
import com.ibm.wala.ipa.callgraph.propagation.PointerAnalysis;
import com.ibm.wala.ipa.cha.ClassHierarchyException;
import com.ibm.wala.ssa.IR;
import com.ibm.wala.ssa.SSAGetInstruction;
import com.ibm.wala.types.MethodReference;
import com.ibm.wala.types.TypeReference;
import com.ibm.wala.util.CancelException;
import com.ibm.wala.util.NullProgressMonitor;
import com.ibm.wala.util.collections.HashMapFactory;
import com.ibm.wala.util.collections.HashSetFactory;

import edu.emory.mathcs.backport.java.util.Collections;

public class TestPandasModel extends TestPythonMLCallGraphShape {

@Test
public void testPandas1() throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException {
PythonAnalysisEngine<TensorTypeAnalysis> engine = builder("pandas1.py");
PythonAnalysisEngine<TensorTypeAnalysis> engine = makeEngine("pandas1.py");
CallGraphBuilder<? extends InstanceKey> builder = engine.defaultCallGraphBuilder();
CallGraph CG = builder.makeCallGraph(engine.getOptions(), new NullProgressMonitor());
PointerAnalysis<? extends InstanceKey> PA = engine.getPointerAnalysis();
Expand Down
35 changes: 29 additions & 6 deletions ...st.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestPythonMLCallGraphShape.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import java.util.Iterator;
import java.util.Set;

import com.ibm.wala.cast.ipa.callgraph.CAstCallGraphUtil;
import com.ibm.wala.cast.python.client.PythonAnalysisEngine;
import com.ibm.wala.cast.python.ml.analysis.TensorTypeAnalysis;
import com.ibm.wala.cast.python.ml.analysis.TensorVariable;
Expand All @@ -14,15 +15,21 @@
import com.ibm.wala.cast.python.types.PythonTypes;
import com.ibm.wala.cast.types.AstMethodReference;
import com.ibm.wala.classLoader.CallSiteReference;
import com.ibm.wala.classLoader.Module;
import com.ibm.wala.classLoader.SourceURLModule;
import com.ibm.wala.ipa.callgraph.CGNode;
import com.ibm.wala.ipa.callgraph.CallGraph;
import com.ibm.wala.ipa.callgraph.CallGraphBuilder;
import com.ibm.wala.ipa.callgraph.propagation.InstanceKey;
import com.ibm.wala.ipa.callgraph.propagation.PropagationCallGraphBuilder;
import com.ibm.wala.ipa.callgraph.propagation.SSAPropagationCallGraphBuilder;
import com.ibm.wala.ipa.cha.ClassHierarchyException;
import com.ibm.wala.ssa.SSAAbstractInvokeInstruction;
import com.ibm.wala.types.MethodReference;
import com.ibm.wala.types.TypeReference;
import com.ibm.wala.util.CancelException;
import com.ibm.wala.util.NullProgressMonitor;
import com.ibm.wala.util.collections.HashSetFactory;

public abstract class TestPythonMLCallGraphShape extends TestPythonCallGraphShape {

Expand All @@ -32,16 +39,17 @@ interface CheckTensorOps {
void check(PropagationCallGraphBuilder cgBuilder, CallGraph CG, TensorTypeAnalysis result);
}

protected PythonAnalysisEngine<TensorTypeAnalysis> builder(String name) throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException {
@Override
protected PythonAnalysisEngine<TensorTypeAnalysis> makeEngine(String... name) throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException {
PythonAnalysisEngine<TensorTypeAnalysis> engine = new PythonTensorAnalysisEngine();
engine.setModuleFiles(Collections.singleton(getScript(name)));
Set<Module> modules = HashSetFactory.make();
for(String n : name) {
modules.add(getScript(n));
}
engine.setModuleFiles(modules);
return engine;
}

protected CallGraph process(String name) throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException {
return builder(name).buildDefaultCallGraph();
}

protected void checkTensorOp(PropagationCallGraphBuilder cgBuilder, CallGraph CG, TensorTypeAnalysis result, String fn, String in, String out) {
boolean found = false;
Set<CGNode> nodes = CG.getNodes(MethodReference.findOrCreate(TypeReference.findOrCreate(PythonTypes.pythonLoader, "Ltensorflow/functions/" + fn), AstMethodReference.fnSelector));
Expand Down Expand Up @@ -78,4 +86,19 @@ protected void checkTensorOps(String url, CheckTensorOps check)
System.err.println(result);
check.check(cgBuilder, CG, result);
}

public static void main(String[] args) throws ClassHierarchyException, IllegalArgumentException, CancelException, IOException {
TestPythonMLCallGraphShape driver = new TestPythonMLCallGraphShape() {

};

PythonAnalysisEngine<?> E = driver.makeEngine(args[0]);

CallGraphBuilder<? super InstanceKey> builder = E.defaultCallGraphBuilder();
CallGraph CG = builder.makeCallGraph(E.getOptions(), new NullProgressMonitor());

CAstCallGraphUtil.AVOID_DUMP = false;
CAstCallGraphUtil.dumpCG(((SSAPropagationCallGraphBuilder)builder).getCFAContextInterpreter(), E.getPointerAnalysis(), CG);
}

}
2 changes: 1 addition & 1 deletion ...wala.cast.python.ml.test/source/com/ibm/wala/cast/python/ml/test/TestTensorflowModel.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public void testTf1() throws ClassHierarchyException, IllegalArgumentException,
check: {
for(CGNode node : nodes) {
for(Iterator<CGNode> ns = CG.getPredNodes(node); ns.hasNext(); ) {
if (ns.next().getMethod().isSynthetic()) {
if (ns.next().getMethod().isWalaSynthetic()) {
break check;
}
}
Expand Down
2 changes: 1 addition & 1 deletion com.ibm.wala.cast.python.ml/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
<dependency>
<groupId>org.eclipse.lsp4j</groupId>
<artifactId>org.eclipse.lsp4j</artifactId>
<version>0.6.0-SNAPSHOT</version>
<version>0.5.0</version>
</dependency>
<dependency>
<groupId>commons-cli</groupId>
Expand Down
6 changes: 6 additions & 0 deletions com.ibm.wala.cast.python.ml/run.sh
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
#!/bin/bash

ME=`realpath $0`
DIR=`dirname $ME`

cat -u | tee -a /tmp/lsp.in.log | $JAVA_HOME/bin/java -Xdebug -Xrunjdwp:transport=dt_socket,address=127.0.0.1:6660,server=y,suspend=n -jar $DIR/target/com.ibm.wala.cast.python.ml-0.0.1-SNAPSHOT.jar --mode stdio | tee -a /tmp/lsp.out.log
Loading

0 comments on commit c9ed3d2

Please sign in to comment.