BIG changes- parser is integrating nicely

Author: kazimuth

BUILD.md

@@ -16,13 +16,12 @@ Check build.xml for building instructions.
 Done:
 - Basic functionality- running python code
 - Basic indentation & highlighting
-- Basic preprocessor
+- Preprocessor
 
 Currently working on:
-- Proper parser for the preprocessor
+- Library imports and sketch exports
 
 Future work:
-- Library imports and sketch exports
 - Better autoindent & syntax highlighting
 - REPL for live coding
 

README.md

@@ -4,7 +4,7 @@
 	<!-- Note: now compiles two separate jars- one for python mode, and one that's a wrapper for the processing core that can run python files -->
 
 	<!-- You'll want to change these to compile. -->
-	<property name="processing.base" 			location="C:\Dev\processing-2.0.1-windows64\processing-2.0.1" /> 	<!-- The directory that you unpacked Processing into. -->
+	<property name="processing.base" 			location="C:\Dev\processing-2.0.2-windows64\processing-2.0.2" /> 	<!-- The directory that you unpacked Processing into. -->
 	<property name="processing.sketchfolder" 	location="C:\Dev\Processing" />										<!-- Your sketchbook folder. -->
 	<property name="java.target.bootclasspath"	location="C:\Program Files\Java\jdk1.6.0_38\jre\lib\rt.jar" />		<!-- If you're using a newer jdk -->
 
@@ -72,8 +72,6 @@
 	<target name="generate" depends="get" unless="gen.present">
 		<mkdir dir="${gen}" />
 		<java jar="tool/antlr-4.1-complete.jar" fork="true">
-			<arg value="-no-listener" />
-			<arg value="-visitor" />
 			<arg value="-lib" />
 			<arg value="${basedir}/${src}/${preproc.path}" />
 			<arg value="-o" />
@@ -116,7 +114,7 @@
 		<mkdir dir="${bundle}/mode" />
 
 		<!-- make mode jar -->
-		<jar jarfile="${bundle}/mode/${lib.name}.jar" basedir="build" includes="${lib.path}/*.class"/>
+		<jar jarfile="${bundle}/mode/${lib.name}.jar" basedir="build" includes="${lib.path}/*.class ${preproc.path}/*.class"/>
 		<!-- make wrapper jar -->
 		<jar jarfile="${bundle}/mode/${wrap.name}.jar" basedir="build" includes="${wrap.path}/*"/> <!-- include *.py -->
 
@@ -161,7 +159,7 @@
 		<delete dir="${dist}" />
 	</target>
 
-	<target name="cleangen" >
+	<target name="cleangen" depends="clean" >
 		<delete dir="${gen}" />
 	</target>
 

build.xml

@@ -2,6 +2,7 @@
 
 import java.io.File;
 import java.io.PrintWriter;
+import java.util.Set;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
@@ -10,18 +11,26 @@
 import processing.app.Sketch;
 import processing.app.SketchCode;
 
+import info.sansgills.mode.python.preproc.*;
+
+import org.antlr.v4.runtime.ANTLRInputStream;
+import org.antlr.v4.runtime.CommonTokenStream;
+import org.antlr.v4.runtime.tree.ParseTree;
+import org.antlr.v4.runtime.tree.ParseTreeWalker;
+
+
 
 /**
  * 
  * Class to handle the building of python files.
- * Given that python runs as an interpreter, all this does is handle preprocessing.
+ * Given that python runs as an interpreter, all this does is handle preprocessing
+ * and write to an output file.
  * 
  * See BUILD.md for notes.
  * 
  */
 
 public class PythonBuild {
-	
 	Sketch sketch;
 	String resultProgram;
 
@@ -30,6 +39,8 @@ public class PythonBuild {
 	File binFolder;
 	File outFile;
 
+	String classPath, pythonLibs, javaLibs;
+	
 	//build tracking
 	private int buildnumber;
 	private static int buildstotal = 0;
@@ -69,62 +80,60 @@ public void build() throws Exception{
 				+ sketch.getName().toLowerCase()
 				+ ".py");
 		outFile.createNewFile();
-	
+		
 
 		PrintWriter writer = new PrintWriter(outFile);
 		writer.print(resultProgram);
 		writer.close();
 	}
 
 
-	
-	//Things we don't need to reload every build
-	//Some regexes
-	//A hack, but much less work than a full parser, and we don't need to do very much
-	private static Pattern getSpecial;		//replace mousePressed / keyPressed /frameRate with getmousePressed, etc.
-	private static Pattern instanceVars;	//replace 'mouseX' with __applet__.mouseX, etc.
-	private static Pattern findSetup, indent;	//for static-mode sketches
-	
-	static{
-		getSpecial = Pattern.compile("(?<!def\\s{1,1000})(mousePressed|keyPressed|frameRate)(?![0-9a-zA-Z_])(?!\\s{0,1000}\\()");
-		instanceVars = Pattern.compile("(mouseX|mouseY|pmouseX|pmouseY|mouseButton|"
-				+"keyCode|key|pixels|width|height|displayWidth|displayHeight|focused|frameCount)(?![0-9a-zA-Z_])");
-		findSetup = Pattern.compile("^def\\s+setup\\s*\\(\\s*\\)\\s*:", Pattern.MULTILINE);
-		indent = Pattern.compile("\r?\n");		
-	}
-	
-	
-	
 	/*
 	 * Turn .pde into valid python
+	 * 
+	 * Now with antlr!
 	 */
 	private String preprocess(StringBuilder program){
+		if(Pattern.matches("\\s*", program)){
+			return "def setup():\n\tpass\n\n"; //empty sketch; TODO fix hack
+		}
 		try{
-			String temp;
+			PyPdeLexer lexer = new PyPdeLexer(new ANTLRInputStream(program.toString()));
+			CommonTokenStream tokens = new CommonTokenStream(lexer);
+			PyPdeParser parser = new PyPdeParser(tokens);
+			ParseTree tree = parser.script();
+			PyPdeConverter converter = new PyPdeConverter(tokens);
+			ParseTreeWalker walker = new ParseTreeWalker();
+			walker.walk(converter, tree);
 
-			Matcher regex = getSpecial.matcher(program);
+			String result = converter.getText();
 
-			temp = regex.replaceAll("get$1()");
+			String[] libraries = converter.getLibraries();
 
-			regex = instanceVars.matcher(temp);
+			Set<String> knownPythonLibraries = PythonMode.getPythonLibraries();
 
-			temp = regex.replaceAll("__applet__.$1");
+			pythonLibs = "";
+			javaLibs = "";
 
-			regex = findSetup.matcher(temp);
-			
-			if(!regex.find()){
-				//no setup function; static mode sketch
-				//TODO handle bad indentation
-				program = new StringBuilder(temp);
-				program.append("\nnoLoop()\n");			//no need to call draw()
-				program.insert(0, "def setup():\n");	//put the whole function in setup()
-				
-				regex = indent.matcher(program);
-				
-				temp = regex.replaceAll("\n\t");		//indent everything a level! (except the first line)
+			for(String lib : libraries){
+				if(knownPythonLibraries.contains(lib)){
+					//it's a python library; for now, I'm only handling ones built into jython, so no need to do anything
+				}else{
+					//normal java library; we can use the normal library infrastructure
+					int dot = lib.lastIndexOf('.');
+				    String entry = (dot == -1) ? lib : lib.substring(0, dot);
+				    Library library = mode.getLibrary(entry);
+				    javaLibs += library.getJarPath() + System.getProperty("path.separator");
+				}
 			}
 
-			return temp;
+			if(pythonLibs.equals("")){
+				pythonLibs = null;
+			}
+			if(javaLibs.equals("")){
+				javaLibs = null;
+			}
+			return result;
 
 		}catch(Exception e){
 			System.err.println("Preprocessing failed.");
@@ -133,13 +142,20 @@ private String preprocess(StringBuilder program){
 		}
 	}
 
+	public String getJavaLibraries(){
+		return javaLibs;
+	}
+	public boolean hasJavaLibraries(){
+		return javaLibs != null;
+	}
 
 
-	
-	
-	
-	
-	
+	public String getPythonLibraries(){
+		return pythonLibs;
+	}
+	public boolean hasPythonLibraries(){
+		return pythonLibs != null;
+	}
 
 	/*
 	 * The output code string
@@ -164,35 +180,13 @@ public int getBuildNumber(){
 
 	/*
 	 * Classes used to run the build.
-	 * 
-	 * TODO build during preprocess
 	 */
 	public String getClassPath() {
-		//the Processing classpath
-		String cp = mode.getCoreLibrary().getClassPath();
-		
-		// From JavaMode.java:
-		// Finally, add the regular Java CLASSPATH. This contains everything
-		// imported by the PDE itself (core.jar, pde.jar, quaqua.jar) which may
-		// in fact be more of a problem.
-		String javaClassPath = System.getProperty("java.class.path");
-		// Remove quotes if any.. A messy (and frequent) Windows problem
-		if (javaClassPath.startsWith("\"") && javaClassPath.endsWith("\"")) {
-			javaClassPath = javaClassPath.substring(1,
-					javaClassPath.length() - 1);
-		}
-		cp += File.pathSeparator + javaClassPath;
-		
-		//The .jars for this mode; Jython and wrapper, primarily
-		String jythonModeLib = Base.getSketchbookModesFolder().getAbsolutePath()
-				+ File.separator
-				+ "PythonMode"
-				+ File.separator
-				+ "mode";
-		cp += Base.contentsToClassPath(new File(jythonModeLib));
-		
-		
-		return cp; // TODO libraries?
+		return classPath; //computed during preprocessing
+	}
+	
+	public String getName() {
+		return sketch.getName();
 	}
 
 	/*
@@ -201,18 +195,4 @@ public String getClassPath() {
 	public String getJavaLibraryPath(){
 		return "";
 	}
-	
-	/*
-	 * The PApplet subclass to extract from the code
-	 */
-	public String getClassName(){
-		return "Placeholder";		//TODO implement naming scheme
-	}
-	
-	/*
-	 * The name of the output object
-	 */
-	public String getObjName(){
-		return "applet";			// TODO
-	}
 }

release/PythonMode.zip

@@ -97,7 +97,7 @@ public boolean keyPressed(KeyEvent event){
 	private static Pattern incIndent = Pattern.compile(":( |\\t)*(#.*)?$"); //TODO fix; breaks on strings (":#"\n) and so on
 
 	String getIndent(int cursor, String text){
-		if(cursor == 1) return "\n";
+		if(cursor <= 1) return "\n";
 
 		int lineStart, lineEnd;
 		int i;