first draft for Flat File Maker

Author: basilwong

.idea/workspace.xml

@@ -4,18 +4,28 @@
 import org.apache.commons.lang3.StringUtils;
 import org.apache.spark.api.java.*;
 import org.apache.spark.sql.*;
-import org.apache.spark.sql.types.*;
 import java.util.*;
-import org.apache.spark.sql.functions;
-
-
+import java.sql.Timestamp;
+import java.text.SimpleDateFormat;
 
 public class FlatFileMaker {
 
-
-
-
-
-
+    public static void genFlatFile(Dataset<Row> data, List<Integer> colSizes) {
+        JavaRDD<String> trip = data.rdd().toJavaRDD().map(row-> rowToFWSTring(colSizes, row));
+        trip.saveAsTextFile(determinePath());
+    }
+
+    private static String rowToFWSTring(List<Integer> rowSize, Row r) {
+        String[] vals = new String[r.size()];
+        for (int i = 0; i < r.size(); i++) {
+            vals[i] = StringUtils.leftPad(r.getString(i), rowSize.get(i), ' ').substring(0, rowSize.get(i));
+        }
+        return String.join("", vals);
+    }
+
+    private static String determinePath() {
+        Timestamp timestamp = new Timestamp(System.currentTimeMillis());
+        return "src/main/resources/" + timestamp.toString();
+    }
 
 }

src/main/java/com/example/FlatFileMaker.java

@@ -42,33 +42,11 @@ public static void main(String[] args) {
               .withColumn("Last-Name", functions.substring(methodDS.col("Last-Name"), 0, 10));
       newDS.show();
 
-//      // Edit Rows
 
-//       Dataset<Row> sinDS = newDS
 
 
-//      StructType sek = new StructType(new StructField[] {
-//                  new StructField("a", DataTypes.createArrayType(DataTypes.StringType), false, Metadata.empty())
-//      });
       JavaRDD<String> trip = newDS.rdd().toJavaRDD().map(row-> rowToFWSTring(parser.getColSizes(), row));
       trip.saveAsTextFile("src/main/resources/outputyo");
-//      Dataset<Row> readyToWrite = sesh.createDataset(trip, sek);
-//      System.out.println(readyToWrite);
-
-//      newDS.select(newDS.col("*"))
-//                      .write().mode(SaveMode.Append).csv("src/main/resources/output");
-//
-//      newDS
-//           .write()
-//           .mode(SaveMode.Append)
-//           .csv("src/main/resources/output");
-
-
-
-
-//              .map(row -> rowToFWSTring(row)).collect();
-
-
 
    }
 }