Vertical col support

common/client/src/main/java/zingg/common/client/ILabelDataViewHelper.java

@@ -1,5 +1,7 @@
 package zingg.common.client;
 
+import zingg.common.client.util.verticalDisplay.VerticalDisplayUtility;
+
 import java.util.List;
 
 public interface ILabelDataViewHelper<S, D, R, C> {
@@ -20,9 +22,10 @@ public interface ILabelDataViewHelper<S, D, R, C> {
 
 	String getMsg2(double prediction, double score);
 
-	void displayRecords(ZFrame<D, R, C> records, String preMessage, String postMessage);
+	void displayRecords(ZFrame<D, R, C> records, String preMessage, String postMessage) throws ZinggClientException;
 
 	void printMarkedRecordsStat(long positivePairsCount, long negativePairsCount, long notSurePairsCount,
 			long totalCount);
 
+	void setVerticalDisplayUtilityNew(VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility);
 }

common/client/src/main/java/zingg/common/client/ZFrame.java

@@ -184,7 +184,4 @@ public interface ZFrame<D, R, C> {
 
 	public C gt(C column1, C column2);
 
-    public ZFrame<D, R, C> transpose(String pivotColumn);
-
-    public void showVertical();
 }

common/client/src/main/java/zingg/common/client/util/verticalDisplay/VerticalDisplayModel.java

@@ -0,0 +1,4 @@
+package zingg.common.client.util.verticalDisplay;
+
+public abstract class VerticalDisplayModel {
+}

common/client/src/main/java/zingg/common/client/util/verticalDisplay/VerticalDisplayTwoRowModel.java

@@ -0,0 +1,13 @@
+package zingg.common.client.util.verticalDisplay;
+
+public class VerticalDisplayTwoRowModel extends VerticalDisplayModel {
+    private final String Field;
+    private final String Value1;
+    private final String Value2;
+
+    public VerticalDisplayTwoRowModel(String Field, String Value1, String Value2) {
+        this.Field = Field;
+        this.Value1 = Value1;
+        this.Value2 = Value2;
+    }
+}

common/client/src/main/java/zingg/common/client/util/verticalDisplay/VerticalDisplayUtility.java

@@ -0,0 +1,45 @@
+package zingg.common.client.util.verticalDisplay;
+
+import zingg.common.client.ZFrame;
+import zingg.common.client.ZinggClientException;
+import zingg.common.client.util.DFObjectUtil;
+import zingg.common.client.util.Pair;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class VerticalDisplayUtility<S, D, R, C> {
+
+    private final DFObjectUtil<S, D, R, C> dfObjectUtil;
+
+    public VerticalDisplayUtility(DFObjectUtil<S, D, R, C> dfObjectUtil) {
+        this.dfObjectUtil = dfObjectUtil;
+    }
+
+    public void showVertical(ZFrame<D, R, C> zFrame) throws ZinggClientException {
+        ZFrame<D, R, C> verticalZFrame = convertVertical(zFrame);
+        verticalZFrame.show(1000);
+    }
+
+    public ZFrame<D, R, C> convertVertical(ZFrame<D, R, C> zFrame) throws ZinggClientException {
+        try {
+            String[] columns = zFrame.columns();
+            List<Pair<String, String>> comparison_pairs = getComparisonPairs(zFrame);
+            List<VerticalDisplayModel> rowList = getList(comparison_pairs, columns);
+            return dfObjectUtil.getDFFromObjectList(rowList, VerticalDisplayTwoRowModel.class);
+        } catch (Exception exception) {
+            throw new ZinggClientException("Error occurred while converting to vertical, " + exception.getMessage());
+        }
+    }
+
+    private List<VerticalDisplayModel> getList(List<Pair<String, String>> comparison_pairs, String[] columns) {
+        List<VerticalDisplayModel> samples = new ArrayList<>();
+        for (int idx = 0; idx < comparison_pairs.size(); idx++){
+            samples.add(new VerticalDisplayTwoRowModel(columns[idx], comparison_pairs.get(idx).getFirst(), comparison_pairs.get(idx).getSecond()));
+        }
+
+        return samples;
+    }
+
+    protected abstract List<Pair<String, String>> getComparisonPairs(ZFrame<D, R, C> zFrame);
+}

common/core/src/main/java/zingg/common/core/executor/LabelDataViewHelper.java

@@ -11,13 +11,15 @@
 import zingg.common.client.ZinggClientException;
 import zingg.common.client.util.ColName;
 import zingg.common.client.util.ColValues;
+import zingg.common.client.util.verticalDisplay.VerticalDisplayUtility;
 import zingg.common.core.context.IContext;
 import zingg.common.core.util.LabelMatchType;
 
 public class LabelDataViewHelper<S,D,R,C,T> extends ZinggBase<S, D, R, C, T> implements ILabelDataViewHelper<S, D, R, C> {
 
 	private static final long serialVersionUID = 1L;
 	public static final Log LOG = LogFactory.getLog(LabelDataViewHelper.class);
+	private VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility;
 
 	public LabelDataViewHelper(IContext<S,D,R,C,T> context, ClientOptions clientOptions) {
 		setContext(context);
@@ -84,11 +86,11 @@ public String getMsg2(double prediction, double score) {
 
 
 	@Override
-	public void displayRecords(ZFrame<D, R, C> records, String preMessage, String postMessage) {
+	public void displayRecords(ZFrame<D, R, C> records, String preMessage, String postMessage) throws ZinggClientException {
 		//System.out.println();
 		System.out.println(preMessage);
 //		records.show(false);
-		records.showVertical();
+		verticalDisplayUtility.showVertical(records);
 		System.out.println(postMessage);
 		System.out.println("\tWhat do you think? Your choices are: ");
 		System.out.println();
@@ -126,6 +128,9 @@ public ILabelDataViewHelper<S, D, R, C> getLabelDataViewHelper() throws Unsuppor
 		return this;
 	}
 
-
+	@Override
+	public void setVerticalDisplayUtilityNew(VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility) {
+		this.verticalDisplayUtility = verticalDisplayUtility;
+	}
 
 }

common/core/src/main/java/zingg/common/core/executor/LabelUpdater.java

@@ -124,7 +124,7 @@
 		return updatedRecords;
 	}
 
-	protected int getUserInput(ZFrame<D,R,C> lines,ZFrame<D,R,C> currentPair,String cluster_id) {
+	protected int getUserInput(ZFrame<D,R,C> lines,ZFrame<D,R,C> currentPair,String cluster_id) throws ZinggClientException {
 //		List<C> displayCols = getDSUtil().getFieldDefColumns(lines, args, false, args.getShowConcise());
 		ZidAndFieldDefSelector zidAndFieldDefSelector = new ZidAndFieldDefSelector(args.getFieldDefinition(), false, args.getShowConcise());
 		int matchFlag = currentPair.getAsInt(currentPair.head(),ColName.MATCH_FLAG_COL);

common/core/src/main/java/zingg/common/core/executor/Labeller.java

@@ -13,6 +13,7 @@
 import zingg.common.client.cols.ZidAndFieldDefSelector;
 import zingg.common.client.options.ZinggOptions;
 import zingg.common.client.util.ColName;
+import zingg.common.client.util.verticalDisplay.VerticalDisplayUtility;
 
 public abstract class Labeller<S,D,R,C,T> extends ZinggBase<S,D,R,C,T> {
 
@@ -137,7 +138,7 @@ public ZFrame<D,R,C> processRecordsCli(ZFrame<D,R,C>  lines) throws ZinggClientE
 	}
 
 
-	protected int displayRecordsAndGetUserInput(ZFrame<D,R,C> records, String preMessage, String postMessage) {
+	protected int displayRecordsAndGetUserInput(ZFrame<D,R,C> records, String preMessage, String postMessage) throws ZinggClientException {
 		getLabelDataViewHelper().displayRecords(records, preMessage, postMessage);
 		int selection = readCliInput();
 		return selection;
@@ -174,6 +175,7 @@ public void setTrainingDataModel(ITrainingDataModel<S, D, R, C> trainingDataMode
 	public ILabelDataViewHelper<S, D, R, C> getLabelDataViewHelper() {
 		if(labelDataViewHelper==null) {
 			labelDataViewHelper = new LabelDataViewHelper<S,D,R,C,T>(getContext(), getClientOptions());
+			labelDataViewHelper.setVerticalDisplayUtilityNew(getVerticalDisplayUtilityNew());
 		}
     	return labelDataViewHelper;
     }
@@ -182,6 +184,7 @@ public void setLabelDataViewHelper(ILabelDataViewHelper<S, D, R, C> labelDataVie
 		this.labelDataViewHelper = labelDataViewHelper;
 	}
 
+	protected abstract VerticalDisplayUtility<S, D, R, C> getVerticalDisplayUtilityNew();
 }
 
 

common/core/src/test/java/zingg/common/core/util/TestVerticalDisplayUtility.java

@@ -0,0 +1,134 @@
+package zingg.common.core.util;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+import zingg.common.client.ZFrame;
+import zingg.common.client.ZinggClientException;
+import zingg.common.client.util.DFObjectUtil;
+import zingg.common.client.util.verticalDisplay.VerticalDisplayUtility;
+
+import java.util.ArrayList;
+import java.util.List;
+
+public abstract class TestVerticalDisplayUtility<S, D, R, C> {
+
+    private final DFObjectUtil<S, D, R, C> dfObjectUtil;
+
+    public TestVerticalDisplayUtility(DFObjectUtil<S, D, R, C> dfObjectUtil) {
+        this.dfObjectUtil = dfObjectUtil;
+    }
+
+    @Test
+    public void testWithoutNulls() throws Exception, ZinggClientException {
+        VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility = getVerticalDisplayUtility();
+        ZFrame<D, R, C> zFrame = dfObjectUtil.getDFFromObjectList(getDataWithoutNulls(), Customer.class);
+        ZFrame<D, R, C> zFrameVertical = verticalDisplayUtility.convertVertical(zFrame);
+
+        //Assert on total number of rows
+        Assertions.assertEquals(5, zFrameVertical.count(), "Count is not as expected");
+
+        //Assert on first row
+        R row = zFrameVertical.head();
+        Assertions.assertEquals("fname", zFrameVertical.getAsString(row, "Field"));
+        Assertions.assertEquals("Nitish", zFrameVertical.getAsString(row, "Value1"));
+        Assertions.assertEquals("Nitis", zFrameVertical.getAsString(row, "Value2"));
+    }
+
+    @Test
+    public void testWithNulls() throws Exception, ZinggClientException {
+        VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility = getVerticalDisplayUtility();
+        ZFrame<D, R, C> zFrame = dfObjectUtil.getDFFromObjectList(getDataWithNulls(), Customer.class);
+        ZFrame<D, R, C> zFrameVertical = verticalDisplayUtility.convertVertical(zFrame);
+
+        //Assert on total number of rows
+        Assertions.assertEquals(5, zFrameVertical.count(), "Count is not as expected");
+
+        //Assert on first row
+        R row = zFrameVertical.head();
+        Assertions.assertEquals("fname", zFrameVertical.getAsString(row, "Field"));
+        Assertions.assertEquals("Nitish", zFrameVertical.getAsString(row, "Value1"));
+        Assertions.assertNull(zFrameVertical.getAsString(row, "Value2"));
+    }
+
+    @Test
+    public void testWithFullRowNullExceptPrimaryKey() throws Exception, ZinggClientException {
+        VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility = getVerticalDisplayUtility();
+        ZFrame<D, R, C> zFrame = dfObjectUtil.getDFFromObjectList(getDataWithFullRowNullExceptPrimaryKey(), Customer.class);
+        ZFrame<D, R, C> zFrameVertical = verticalDisplayUtility.convertVertical(zFrame);
+
+        //Assert on total number of rows
+        Assertions.assertEquals(5, zFrameVertical.count(), "Count is not as expected");
+
+        //Assert on first row
+        R row = zFrameVertical.head();
+        Assertions.assertEquals("fname", zFrameVertical.getAsString(row, "Field"));
+        Assertions.assertEquals("Nitish", zFrameVertical.getAsString(row, "Value1"));
+        Assertions.assertNull(zFrameVertical.getAsString(row, "Value2"));
+    }
+
+    @Test
+    public void testWithFullRowNull() throws Exception, ZinggClientException {
+        VerticalDisplayUtility<S, D, R, C> verticalDisplayUtility = getVerticalDisplayUtility();
+        ZFrame<D, R, C> zFrame = dfObjectUtil.getDFFromObjectList(getDataWithFullRowNull(), Customer.class);
+        ZFrame<D, R, C> zFrameVertical = verticalDisplayUtility.convertVertical(zFrame);
+
+        //Assert on total number of rows
+        Assertions.assertEquals(5, zFrameVertical.count(), "Count is not as expected");
+
+        //Assert on first row
+        R row = zFrameVertical.head();
+        Assertions.assertEquals("fname", zFrameVertical.getAsString(row, "Field"));
+        Assertions.assertEquals("Nitish", zFrameVertical.getAsString(row, "Value1"));
+        Assertions.assertNull(zFrameVertical.getAsString(row, "Value2"));
+    }
+
+
+
+
+    public static List<Customer> getDataWithoutNulls() {
+        List<Customer> sample = new ArrayList<Customer>();
+        sample.add(new Customer("Nitish", "Joshi", "123", "0000", "1"));
+        sample.add(new Customer("Nitis", "Joshi", "123", "0000", "2"));
+        return sample;
+    }
+
+    public static List<Customer> getDataWithNulls() {
+        List<Customer> sample = new ArrayList<Customer>();
+        sample.add(new Customer("Nitish", "Joshi", "123", null, "1"));
+        sample.add(new Customer(null, null, "123", "0000", "2"));
+        return sample;
+    }
+
+    public static List<Customer> getDataWithFullRowNullExceptPrimaryKey() {
+        List<Customer> sample = new ArrayList<Customer>();
+        sample.add(new Customer("Nitish", "Joshi", "123", null, "1"));
+        sample.add(new Customer(null, null, null, null, "2"));
+        return sample;
+    }
+
+    public static List<Customer> getDataWithFullRowNull() {
+        List<Customer> sample = new ArrayList<Customer>();
+        sample.add(new Customer("Nitish", "Joshi", "123", null, "1"));
+        sample.add(new Customer(null, null, null, null, null));
+        return sample;
+    }
+
+
+    public static class Customer {
+        public final String fname;
+        public final String lname;
+        public final String ssn;
+        public final String dob;
+        public final String row_uuid;
+
+        public Customer(String fname, String lname, String ssn, String dob, String row_uuid) {
+            this.fname = fname;
+            this.lname = lname;
+            this.ssn = ssn;
+            this.dob = dob;
+            this.row_uuid = row_uuid;
+        }
+    }
+
+    protected abstract VerticalDisplayUtility<S, D, R, C> getVerticalDisplayUtility();
+}

python/zingg/client.py

@@ -20,7 +20,6 @@
 _spark_ctxt = None
 _sqlContext = None
 _spark = None
-<<<<<<< HEAD
 _zingg_jar = 'zingg-0.4.1-SNAPSHOT.jar'
 
 

spark/client/src/main/java/zingg/spark/client/SparkClient.java

@@ -81,7 +81,7 @@ public SparkSession getSession() {
                     .builder()
                     .appName("Zingg")
                     .getOrCreate();	
-			JavaSparkContext ctx = JavaSparkContext.fromSparkContext(session.sparkContext());
+			JavaSparkContext ctx = JavaSparkContext.fromSparkContext(s.sparkContext());
 					JavaSparkContext.jarOfClass(IZingg.class);
 					LOG.debug("Context " + ctx.toString());
 					//initHashFns();