Merge pull request #210 from opensource9ja/fix/rm-nans

Fixes #200 remove NaNs before computing mean or sum

Author: risenW

danfojs-browser/lib/bundle.js

@@ -392,16 +392,31 @@ export class DataFrame extends Ndframe {
    * @returns {Series}
    */
   mean(axis = 1) {
-    if (this.__frame_is_compactible_for_operation) {
-      //check if all types are numeric
-      let operands = this.__get_tensor_and_idx(this, axis);
-      let tensor_vals = operands[0];
-      let idx = operands[1];
-      let result = tensor_vals.mean(operands[2]);
-      let sf = new Series(result.arraySync(), { index: idx });
+    if (this.__frame_is_compactible_for_operation()) {
+      let values;
+      let val_mean = [];
+      if (axis == 1) {
+        values = this.col_data;
+      } else {
+        values = this.values;
+      }
+
+      values.map((arr) => {
+        let temp = utils._remove_nans(arr);
+        let temp_mean = tf.tensor(temp).mean().arraySync();
+        val_mean.push(Number(temp_mean.toFixed(5)));
+      });
+
+      let new_index;
+      if (axis == 1) {
+        new_index = this.column_names;
+      } else {
+        new_index = this.index;
+      }
+      let sf = new Series(val_mean, { columns: "sum", index: new_index });
       return sf;
     } else {
-      throw Error("TypeError: Dtypes of columns must be Float of Int");
+      throw Error("Dtype Error: Operation can not be performed on string type");
     }
   }
 
@@ -904,7 +919,8 @@ export class DataFrame extends Ndframe {
       }
 
       values.map((arr) => {
-        let temp_sum = tf.tensor(arr).sum().arraySync();
+        let temp = utils._remove_nans(arr);
+        let temp_sum = tf.tensor(temp).sum().arraySync();
         val_sums.push(Number(temp_sum.toFixed(5)));
       });
 

danfojs-browser/lib/bundle.js.map

@@ -302,10 +302,12 @@ export class Series extends NDframe {
   sum() {
     utils._throw_str_dtype_error(this, 'sum');
     if (this.dtypes[0] == "boolean") {
-      let temp_sum = this.row_data_tensor.sum().arraySync();
+      let temp = utils._remove_nans(this.values);
+      let temp_sum = tf.tensor(temp).sum().arraySync();
       return Number(temp_sum);
     }
-    let temp_sum = this.row_data_tensor.sum().arraySync();
+    let temp = utils._remove_nans(this.values);
+    let temp_sum = tf.tensor(temp).sum().arraySync();
     return Number(temp_sum.toFixed(5));
   }
 

danfojs-browser/src/core/frame.js

@@ -104,7 +104,7 @@ export class Utils {
     } else {
       var selected = new Set();
       // eslint-disable-next-line no-empty
-      while (selected.add((Math.random() * n) | 0).size < k) {}
+      while (selected.add((Math.random() * n) | 0).size < k) { }
       // eslint-disable-next-line no-undef
       return Array.prototype.map.call(selected, (i) => population[i]);
     }
@@ -210,15 +210,15 @@ export class Utils {
     }
   }
 
-  __checker(arr_val){
+  __checker(arr_val) {
     let dtypes = [];
     let lim;
     let int_tracker = [];
     let float_tracker = [];
     let string_tracker = [];
     let bool_tracker = [];
 
-    if (arr_val.length == 0){
+    if (arr_val.length == 0) {
       dtypes.push("string");
     }
 
@@ -234,7 +234,7 @@ export class Utils {
         int_tracker.push(false);
         string_tracker.push(false);
         bool_tracker.push(true);
-      } else if (isNaN(ele) && typeof ele != "string"){
+      } else if (isNaN(ele) && typeof ele != "string") {
         float_tracker.push(true);
         int_tracker.push(false);
         string_tracker.push(false);
@@ -624,6 +624,21 @@ export class Utils {
     return values;
   }
 
+  /**
+* Replace NaN with null before tensor operations
+* @param {*} arr
+*/
+  _replace_nan_with_null(arr) {
+    let values = arr.map((val) => {
+      if (isNaN(val)) {
+        return null;
+      } else {
+        return val;
+      }
+    });
+    return values;
+  }
+
   __get_duplicate(arr) {
     let temp_obj = {};
     let rslt_obj = {};

danfojs-browser/src/core/series.js

@@ -627,6 +627,11 @@ describe("DataFrame", function () {
       let df = new dfd.DataFrame(data);
       assert.deepEqual(df.mean(0).values, [ 2, 300 ]);
     });
+    it("Removes NaN before calculating mean of a DataFrame", function () {
+      let data = [ [ 11, 20, 3 ], [ NaN, 15, 6 ], [ 2, 30, 40 ], [ 2, 89, 78 ] ];
+      let df = new dfd.DataFrame(data);
+      assert.deepEqual(df.mean().values, [ 5, 38.5, 31.75 ]);
+    });
   });
 
   describe("median", function () {
@@ -1927,7 +1932,12 @@ describe("DataFrame", function () {
       let res = [ 1, 2, 1, 1 ];
       assert.deepEqual(df.sum().values, res);
     });
-
+    it("Sum values of a df with missing values", function () {
+      let data1 = [ [ 11, 20, 3 ], [ null, 15, 6 ], [ 2, 30, 40 ], [ 2, 89, 78 ] ];
+      let df = new dfd.DataFrame(data1);
+      let res = [ 15, 154, 127 ];
+      assert.deepEqual(df.sum().values, res);
+    });
   });
 
   describe("abs", function () {

danfojs-browser/src/core/utils.js

@@ -289,6 +289,11 @@ describe("Series", function () {
       let sf = new dfd.Series(data1);
       assert.deepEqual(sf.mean(), 19.625);
     });
+    it("Computes the mean of elements in a float series with NaN", function () {
+      let data1 = [ 30.1, 40.2, 3.1, 5.1, NaN ];
+      let sf = new dfd.Series(data1);
+      assert.deepEqual(sf.mean(), 19.625);
+    });
     it("Throws error if dtype is string", function () {
       let data1 = [ "boy", "girl", "Man" ];
       let sf = new dfd.Series(data1);
@@ -331,6 +336,11 @@ describe("Series", function () {
       let sf = new dfd.Series(data1);
       assert.deepEqual(sf.sum(), 2);
     });
+    it("Sum values a Series with missing values", function () {
+      let data1 = [ 11, NaN, 2, 2 ];
+      let sf = new dfd.Series(data1);
+      assert.deepEqual(sf.sum(), 15);
+    });
   });
 
   describe("mode", function () {

danfojs-browser/tests/core/frame.js

@@ -330,18 +330,37 @@ class DataFrame extends _generic.default {
   }
 
   mean(axis = 1) {
-    if (this.__frame_is_compactible_for_operation) {
-      let operands = this.__get_tensor_and_idx(this, axis);
+    if (this.__frame_is_compactible_for_operation()) {
+      let values;
+      let val_mean = [];
 
-      let tensor_vals = operands[0];
-      let idx = operands[1];
-      let result = tensor_vals.mean(operands[2]);
-      let sf = new _series.Series(result.arraySync(), {
-        index: idx
+      if (axis == 1) {
+        values = this.col_data;
+      } else {
+        values = this.values;
+      }
+
+      values.map(arr => {
+        let temp = utils._remove_nans(arr);
+
+        let temp_mean = tf.tensor(temp).mean().arraySync();
+        val_mean.push(Number(temp_mean.toFixed(5)));
+      });
+      let new_index;
+
+      if (axis == 1) {
+        new_index = this.column_names;
+      } else {
+        new_index = this.index;
+      }
+
+      let sf = new _series.Series(val_mean, {
+        columns: "sum",
+        index: new_index
       });
       return sf;
     } else {
-      throw Error("TypeError: Dtypes of columns must be Float of Int");
+      throw Error("Dtype Error: Operation can not be performed on string type");
     }
   }
 
@@ -747,7 +766,9 @@ class DataFrame extends _generic.default {
       }
 
       values.map(arr => {
-        let temp_sum = tf.tensor(arr).sum().arraySync();
+        let temp = utils._remove_nans(arr);
+
+        let temp_sum = tf.tensor(temp).sum().arraySync();
         val_sums.push(Number(temp_sum.toFixed(5)));
       });
       let new_index;
@@ -783,7 +804,12 @@ class DataFrame extends _generic.default {
     let tensor_vals, idx, t_axis;
 
     if (axis == 1) {
-      tensor_vals = df.row_data_tensor;
+      let temp_tensor_vals = df.row_data_tensor;
+      let flat_tensor_array = tf.util.flatten(temp_tensor_vals.arraySync());
+
+      const flat_tensor_array_without_nans = utils._replace_nan_with_null(flat_tensor_array);
+
+      tensor_vals = tf.tensor(flat_tensor_array_without_nans, temp_tensor_vals.shape);
       idx = df.column_names;
       t_axis = 0;
     } else {

danfojs-browser/tests/core/series.js

@@ -241,11 +241,15 @@ class Series extends _generic.default {
     utils._throw_str_dtype_error(this, 'sum');
 
     if (this.dtypes[0] == "boolean") {
-      let temp_sum = this.row_data_tensor.sum().arraySync();
+      let temp = utils._remove_nans(this.values);
+
+      let temp_sum = tf.tensor(temp).sum().arraySync();
       return Number(temp_sum);
     }
 
-    let temp_sum = this.row_data_tensor.sum().arraySync();
+    let temp = utils._remove_nans(this.values);
+
+    let temp_sum = tf.tensor(temp).sum().arraySync();
     return Number(temp_sum.toFixed(5));
   }
 

danfojs-node/dist/core/frame.js

@@ -552,6 +552,17 @@ class Utils {
     return values;
   }
 
+  _replace_nan_with_null(arr) {
+    let values = arr.map(val => {
+      if (isNaN(val)) {
+        return null;
+      } else {
+        return val;
+      }
+    });
+    return values;
+  }
+
   __get_duplicate(arr) {
     let temp_obj = {};
     let rslt_obj = {};