Updates to CSV code to avoid java heap space errors.

Change-Id: Idce3cf04e0e0806ef32d87cd1e380e4e860db7fd
WeaveTeam · Oct 31, 2012 · d78ee98 · d78ee98
1 parent 391d118
commit d78ee98
Show file tree

Hide file tree

Showing 12 changed files with 123 additions and 145 deletions.
diff --git a/WeaveAdmin/src/AdminConsole.mxml b/WeaveAdmin/src/AdminConsole.mxml
@@ -20,15 +20,6 @@
 */
 -->
 <mx:Application xmlns:mx="http://www.adobe.com/2006/mxml" layout="absolute" backgroundColor="#7B96B6" creationComplete="handleCreationComplete()" creationPolicy="all" xmlns:managers="weave.ui.admin.managers.*" xmlns:admin="weave.ui.admin.*" xmlns:ui="weave.ui.*">
-	<mx:Script>
-		<![CDATA[
-			import flash.desktop.Clipboard;
-			import flash.desktop.ClipboardFormats;
-			
-			import weave.ui.DraggablePanel;
-			import weave.ui.TranslationPanel;
-		]]>
-	</mx:Script>
 	<!--  cornerRadius="0" -->
 	<mx:Panel
 		headerHeight="25"
@@ -106,26 +97,30 @@
 
 	<mx:Script>
 		<![CDATA[
-			import weave.ui.SessionStateEditor;
-			import weave.api.reportError;
-			import weave.api.WeaveAPI;
-			import weave.data.CSVParser;
+			import flash.desktop.Clipboard;
+			import flash.desktop.ClipboardFormats;
+			
+			import mx.binding.utils.BindingUtils;
+			import mx.controls.Alert;
+			import mx.controls.TextArea;
+			import mx.managers.PopUpManager;
+			
+			import weave.Strings;
+			import weave.Weave;
+			import weave.api.WeaveAPI;
+			import weave.api.reportError;
+			import weave.services.AdminInterface;
+			import weave.services.WeaveAdminService;
+			import weave.ui.DraggablePanel;
+			import weave.ui.SessionStateEditor;
+			import weave.ui.TranslationPanel;
+			import weave.ui.WeaveProgressBar;
+			import weave.ui.admin.LoginPopup;
+			import weave.ui.admin.WizardPanel;
+			import weave.ui.admin.wizards.CSVImportWizard;
+			import weave.ui.admin.wizards.SHPImportWizard;
+			import weave.ui.admin.wizards.SQLImportWizard;
 			import weave.utils.VectorUtils;
-			import mx.binding.utils.BindingUtils;
-			import mx.controls.Alert;
-			import mx.controls.TextArea;
-			import mx.managers.PopUpManager;
-			
-			import weave.Strings;
-			import weave.Weave;
-			import weave.services.AdminInterface;
-			import weave.services.WeaveAdminService;
-			import weave.ui.WeaveProgressBar;
-			import weave.ui.admin.LoginPopup;
-			import weave.ui.admin.WizardPanel;
-			import weave.ui.admin.wizards.CSVImportWizard;
-			import weave.ui.admin.wizards.SHPImportWizard;
-			import weave.ui.admin.wizards.SQLImportWizard;
 			
 			SessionStateEditor.getInstance(); // adds keyboard shortcut f8 to open console
 			

diff --git a/WeaveAdmin/src/weave/services/WeaveAdminService.as b/WeaveAdmin/src/weave/services/WeaveAdminService.as
@@ -365,12 +365,6 @@ package weave.services
 			//query.addAsyncResponder(alertResult);
 			return query;
 		}
-		public function getCSVStringData(csvFile:String):DelayedAsyncInvocation
-		{
-			var query:DelayedAsyncInvocation = invokeAdminService("getCSVStringData", arguments);
-			//query.addAsyncResponder(alertResult);
-			return query;
-		}
 		public function listDBFFileColumns(dbfFileName:String):DelayedAsyncInvocation
 		{
 		    var query:DelayedAsyncInvocation = invokeAdminService("listDBFFileColumns", arguments);

diff --git a/WeaveAdmin/src/weave/ui/admin/wizards/CSVImportWizard.mxml b/WeaveAdmin/src/weave/ui/admin/wizards/CSVImportWizard.mxml
@@ -226,7 +226,6 @@
 				finishEnabled = false;
 				
 				AdminInterface.instance.service.getCSVColumnNames(tableName).addAsyncResponder(handleGetCSVColumnNames,handleCSVFault,tableName);
-//				AdminInterface.instance.service.getCSVStringData(tableName).addAsyncResponder(handleGetCSVStringData,handleCSVFault);
 				
 				csvFileNameInput.text = tableName;
 				tableName = tableName.substring(0,tableName.lastIndexOf("."));

diff --git a/WeaveMobileClient/.actionScriptProperties b/WeaveMobileClient/.actionScriptProperties
@@ -53,9 +53,9 @@
           <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/spark_dmv.swc" useDefaultLinkType="false"/>
           <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/automation.swc" useDefaultLinkType="false"/>
           <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/automation_dmv.swc" useDefaultLinkType="false"/>
-          <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/automation_flashflexkit.swc" useDefaultLinkType="false"/>
-          <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/air/applicationupdater_ui.swc" useDefaultLinkType="false"/>
           <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/air/airframework.swc" useDefaultLinkType="false"/>
+          <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/air/applicationupdater_ui.swc" useDefaultLinkType="false"/>
+          <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/automation_flashflexkit.swc" useDefaultLinkType="false"/>
           <libraryPathEntry kind="3" linkType="1" path="${PROJECT_FRAMEWORKS}/libs/automation_agent.swc" useDefaultLinkType="false"/>
         </excludedEntries>
       </libraryPathEntry>
@@ -93,10 +93,10 @@
   <flashCatalyst validateFlashCatalystCompatibility="false"/>
   <buildTargets>
     <buildTarget buildTargetName="com.adobe.flexide.multiplatform.ios.platform" iosSettingsVersion="1" provisioningFile="d:/Dropbox/weave-mobile/Sanjay_Weave_IOS.mobileprovision" releasePackageType="ipa-ad-hoc">
+      <multiPlatformSettings enabled="true" includePlatformLibs="false" platformID="com.adobe.flexide.multiplatform.ios.platform" version="2"/>
       <airSettings airCertificatePath="D:/Dropbox/weave-mobile/iphone_dev.p12" airTimestamp="true" version="1">
         <airExcludes/>
       </airSettings>
-      <multiPlatformSettings enabled="true" includePlatformLibs="false" platformID="com.adobe.flexide.multiplatform.ios.platform" version="2"/>
       <actionScriptSettings version="1"/>
     </buildTarget>
     <buildTarget buildTargetName="com.qnx.flexide.multiplatform.qnx.platform">
@@ -107,10 +107,10 @@
       <actionScriptSettings version="1"/>
     </buildTarget>
     <buildTarget androidSettingsVersion="1" buildTargetName="com.adobe.flexide.multiplatform.android.platform">
+      <multiPlatformSettings enabled="true" includePlatformLibs="false" platformID="com.adobe.flexide.multiplatform.android.platform" version="2"/>
       <airSettings airCertificatePath="d:/Dropbox/weave-mobile/my-mobile-cert.p12" airTimestamp="true" version="1">
         <airExcludes/>
       </airSettings>
-      <multiPlatformSettings enabled="true" includePlatformLibs="false" platformID="com.adobe.flexide.multiplatform.android.platform" version="2"/>
       <actionScriptSettings version="1"/>
     </buildTarget>
     <buildTarget buildTargetName="default">

diff --git a/WeaveServices/src/weave/servlets/AdminService.java b/WeaveServices/src/weave/servlets/AdminService.java
@@ -703,7 +703,7 @@ public AttributeColumnInfo[] testAllQueries(String connectionName, String passwo
 
 				if (sqlParams != null && sqlParams.length() > 0)
 				{
-					String[] sqlParamsArray = CSVParser.defaultParser.parseCSV(sqlParams)[0];
+					String[] sqlParamsArray = CSVParser.defaultParser.parseCSV(sqlParams, true)[0];
 					result = SQLConfigUtils.getRowSetFromQuery(config, attributeColumnInfo.connection, query, sqlParamsArray);
 				}
 				else
@@ -1076,7 +1076,7 @@ public String[] getCSVColumnNames(String csvFile) throws RemoteException
 		{
 			BufferedReader in = new BufferedReader(new FileReader(new File(uploadPath, csvFile)));
 			String header = in.readLine();
-			String[][] rows = CSVParser.defaultParser.parseCSV(header);
+			String[][] rows = CSVParser.defaultParser.parseCSV(header, true);
 			headerLine = rows[0];
 		}
 		catch (FileNotFoundException e)
@@ -1088,32 +1088,6 @@ public String[] getCSVColumnNames(String csvFile) throws RemoteException
 			throw new RemoteException(e.getMessage());
 		}
 
-		/*
-		 * Old code
-		 * 
-			try
-			{
-				String csvData = org.apache.commons.io.FileUtils.readFileToString(new File(uploadPath, csvFile));
-				// Read first line only (header line).
-				int index = csvData.indexOf("\r");
-				int index2 = csvData.indexOf("\n");
-				if (index2 < index && index2 >= 0)
-					index = index2;
-				String header = index < 0 ? csvData : csvData.substring(0, index);
-				csvData = null; // don't need this in memory anymore
-				String[][] rows = CSVParser.defaultParser.parseCSV(header);
-				headerLine = rows[0];
-			}
-			catch (FileNotFoundException e)
-			{
-				throw new RemoteException(e.getMessage());
-			}
-			catch (Exception e)
-			{
-				throw new RemoteException(e.getMessage());
-			}
-		*/
-
 		return headerLine;
 	}
 
@@ -1147,9 +1121,7 @@ public Boolean checkKeyColumnForCSVImport(String csvFile,String keyColumn,String
 				}
 			}
 
-			String csvData = org.apache.commons.io.FileUtils.readFileToString(new File(uploadPath, csvFile));
-
-			String[][] rows = CSVParser.defaultParser.parseCSV(csvData);
+			String[][] rows = CSVParser.defaultParser.parseCSV(new File(uploadPath, csvFile), true);
 
 			HashMap<String, Boolean> map = new HashMap<String, Boolean>();
 
@@ -1208,35 +1180,6 @@ public Boolean checkKeyColumnForCSVImport(String csvFile,String keyColumn,String
 
 		return isUnique;
 	}
-
-
-	/**
-	 * Read a csv file and return the csv data string .
-	 * 
-	 * @param A csv file name
-	 *            
-	 * @return the csv data string 
-	 * 
-	 */
-	public String getCSVStringData(String csvFile) throws RemoteException
-	{
-		String csvString = null;
-
-		try
-		{
-			csvString = org.apache.commons.io.FileUtils.readFileToString(new File(uploadPath, csvFile));
-		}
-		catch (FileNotFoundException e)
-		{
-			throw new RemoteException(e.getMessage());
-		}
-		catch (Exception e)
-		{
-			throw new RemoteException(e.getMessage());
-		}
-
-		return csvString;
-	}
 
 	public String[] listDBFFileColumns(String dbfFileName) throws RemoteException
 	{
@@ -1431,9 +1374,7 @@ synchronized public String importCSV(String connectionName, String password, Str
 
 			boolean ignoreKeyColumnQueries = false;
 
-			String csvData = org.apache.commons.io.FileUtils.readFileToString(new File(uploadPath, csvFile),"ISO-8859-1");
-
-			String[][] rows = CSVParser.defaultParser.parseCSV(csvData);
+			String[][] rows = CSVParser.defaultParser.parseCSV(new File(uploadPath, csvFile), true);
 
 			if (rows.length == 0)
 				throw new RemoteException("CSV file is empty: " + csvFile);
@@ -1611,10 +1552,7 @@ synchronized public String importCSV(String connectionName, String password, Str
 			}
 			// save modified CSV
 //			BufferedWriter out = new BufferedWriter(new FileWriter(formatted_CSV_path));
-			File out = new File(formatted_CSV_path);
-
-			String temp = SQLUtils.generateCSV(conn, rows);
-			org.apache.commons.io.FileUtils.writeStringToFile(out, temp, "ISO-8859-1");
+			SQLUtils.generateCSV(conn, rows, new File(formatted_CSV_path));
 
 			// Import the CSV file into SQL.
 			// Drop the table if it exists.
@@ -1725,6 +1663,7 @@ synchronized private String addConfigDataTable(
 			String[] filterColumnNames
 		) throws RemoteException
 	{
+		String failMessage = String.format("Failed to add DataTable \"%s\" to the configuration.\n", configDataTableName);
 		if (sqlColumnNames == null || sqlColumnNames.length == 0)
 			throw new RemoteException("No columns were found.");
 		ConnectionInfo info = config.getConnectionInfo(connectionName);
@@ -1855,11 +1794,15 @@ synchronized private String addConfigDataTable(
 		}
 		catch (SQLException e)
 		{
-			throw new RemoteException(String.format("Failed to add DataTable \"%s\" to the configuration.\n", configDataTableName), e);
+			throw new RemoteException(failMessage, e);
 		}
 		catch (RemoteException e)
 		{
-			throw new RemoteException(String.format("Failed to add DataTable \"%s\" to the configuration.\n", configDataTableName), e);
+			throw new RemoteException(failMessage, e);
+		}
+		catch (IOException e)
+		{
+			throw new RemoteException(failMessage, e);
 		}
 
 		return String.format("DataTable \"%s\" was added to the configuration with %s generated attribute column queries.\n", configDataTableName, titles.size());

diff --git a/WeaveServices/src/weave/servlets/DataService.java b/WeaveServices/src/weave/servlets/DataService.java
@@ -207,7 +207,7 @@ public WeaveRecordList getRows(String keyType, String[] keysArray) throws Remote
 				SQLResult result;
 				if (sqlParams != null && sqlParams.length() > 0)
 				{
-					String[] sqlParamsArray = CSVParser.defaultParser.parseCSV(sqlParams)[0];
+					String[] sqlParamsArray = CSVParser.defaultParser.parseCSV(sqlParams, true)[0];
 					result = SQLConfigUtils.getRowSetFromQuery(config, info.connection, sqlQuery, sqlParamsArray);
 				}
 				else
@@ -364,7 +364,7 @@ public AttributeColumnDataWithKeys getAttributeColumn(Map<String, String> params
 
 			if (sqlParams != null && sqlParams.length() > 0)
 			{
-				String[] args = CSVParser.defaultParser.parseCSV(sqlParams)[0];
+				String[] args = CSVParser.defaultParser.parseCSV(sqlParams, true)[0];
 				result = SQLConfigUtils.getRowSetFromQuery(config, info.connection, dataWithKeysQuery, args);
 			}
 			else

diff --git a/WeaveServices/src/weave/servlets/GenericServlet.java b/WeaveServices/src/weave/servlets/GenericServlet.java
@@ -446,7 +446,7 @@ else if (expectedArgTypes[index] == boolean.class || expectedArgTypes[index] ==
 					}
 					else if (expectedArgTypes[index] == String[].class || expectedArgTypes[index] == List.class)
 					{
-						String[][] table = CSVParser.defaultParser.parseCSV((String)value);
+						String[][] table = CSVParser.defaultParser.parseCSV((String)value, true);
 						if (table.length == 0)
 							value = new String[0]; // empty
 						else

diff --git a/WeaveServices/src/weave/tests/TestCSV.java b/WeaveServices/src/weave/tests/TestCSV.java
@@ -20,7 +20,6 @@
 
 import java.io.File;
 
-import org.apache.commons.io.FileUtils;
 import weave.utils.CSVParser;
 
 public class TestCSV
@@ -33,13 +32,9 @@ public static void main(String[] args) throws Exception
 	{
 		File file = new File("src/weave/tests/test.csv");
 		System.out.println(file.getAbsolutePath());
-		String csvData = FileUtils.readFileToString(file);
-
-		System.out.println(csvData);
-		System.out.println();
 
 		System.out.println("parseCSV(csvData)");
-		String[][] parsedRows = CSVParser.defaultParser.parseCSV(csvData);
+		String[][] parsedRows = CSVParser.defaultParser.parseCSV(file, true);
 		for (int i = 0; i < parsedRows.length; i++)
 		{
 			for (int j = 0; j < parsedRows[i].length; j++)
@@ -48,7 +43,7 @@ public static void main(String[] args) throws Exception
 		System.out.println("Compare these results with Microsoft Excel\n");
 
 		System.out.println("parseCSV(csvData, false), parseCSVToken, createCSVToken, parseCSVToken");
-		String[][] rows = CSVParser.defaultParser.parseCSV(csvData, true); // assuming this isn't sql server
+		String[][] rows = CSVParser.defaultParser.parseCSV(file, true); // assuming this isn't sql server
 		for (int i = 0; i < rows.length; i++)
 		{
 			for (int j = 0; j < rows[i].length; j++)