feat(JSON): add skip_null_fields option to omit null-valued record fields

mops.toml

@@ -1,6 +1,6 @@
 [package]
-name = "serde"
-version = "3.5.0"
+name = "serde-core"
+version = "0.1.4"
 description = "A serialisation and deserialisation library for Motoko."
 repository = "https://github.com/NatLabs/serde"
 keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ]

src/CBOR/lib.mo

@@ -93,7 +93,14 @@ module {
                 for ((key, val) in records.vals()) {
                     let res = transpile_candid_to_cbor(val, options);
                     let #ok(cbor_val) = res else return Utils.send_error(res);
-                    newRecords.add((#majorType3(key), cbor_val));
+                    // With `skip_null_fields`, entries whose value encodes to
+                    // CBOR null are treated as "field absent" — same rationale
+                    // as the JSON encoder: many external APIs reject explicit
+                    // null-valued optional fields.
+                    switch (options.skip_null_fields, cbor_val) {
+                        case (true, #majorType7(#_null)) ();
+                        case _ newRecords.add((#majorType3(key), cbor_val));
+                    };
                 };
 
                 #majorType5(Buffer.toArray(newRecords));

src/Candid/Blob/Decoder.mo

@@ -371,7 +371,11 @@ module {
                 case (null) {};
             };
 
-            if (Set.has(visited, nhash, pos) and not Set.has(is_recursive_set, nhash, pos)) {
+            if (Set.has(is_recursive_set, nhash, pos)) {
+                return #Recursive(pos);
+            };
+
+            if (Set.has(visited, nhash, pos)) {
                 ignore Set.put(is_recursive_set, nhash, pos);
                 return #Recursive(pos);
             };

src/Candid/Types.mo

@@ -123,6 +123,14 @@ module {
         /// Must call `Candid.formatCandidTypes` before passing in the types
         types : ?[CandidType];
 
+        /// When encoding records/maps to JSON, omit entries whose value
+        /// resolves to `null` (typically from `?T = null` option fields).
+        /// Many external APIs reject `"field": null` payloads where the
+        /// field is optional and the caller didn't provide a value; this
+        /// flag produces a "field absent" shape instead.
+        /// Default: `false` (preserves previous behaviour).
+        skip_null_fields : Bool;
+
     };
 
     public type ICRC3Value = {
@@ -141,6 +149,8 @@ module {
 
         types = null;
 
+        skip_null_fields = false;
+
     };
 
 };

src/Candid/lib.mo

@@ -36,7 +36,7 @@ module {
     public let repIndyHash = RepIndyHash.hash;
 
     /// Converts a [Candid](#Candid) value to a motoko value
-    public let { decode } = CandidDecoder;
+    public let { decode; decodeOne } = CandidDecoder;
 
     public let Encoder = CandidEncoder;
     public let Decoder = CandidDecoder;

src/JSON/ToText.mo

@@ -1,4 +1,6 @@
 import Buffer "mo:base@0.16/Buffer";
+import Char "mo:core@2.4/Char";
+import Nat32 "mo:core@2.4/Nat32";
 import Result "mo:core@2.4/Result";
 import Text "mo:core@2.4/Text";
 
@@ -15,29 +17,70 @@ module {
     type Candid = Candid.Candid;
     type Result<A, B> = Result.Result<A, B>;
 
+    // Escape a Text value for inclusion in a JSON string literal,
+    // per RFC 8259 §7. Order matters: backslash MUST be escaped
+    // first — every later replacement emits a `\`, and a final
+    // backslash pass would re-double those new backslashes.
+    func escapeJSONString(s : Text) : Text {
+        let chained =
+            Text.replace(s, #text "\\", "\\\\")
+            |> Text.replace(_, #text "\"", "\\\"")
+            |> Text.replace(_, #text "\n", "\\n")
+            |> Text.replace(_, #text "\r", "\\r")
+            |> Text.replace(_, #text "\t", "\\t")
+            |> Text.replace(_, #text "\u{08}", "\\b")
+            |> Text.replace(_, #text "\u{0c}", "\\f");
+        // Remaining U+0000..U+001F (minus the named ones above) → \u00XX.
+        let buf = Buffer.Buffer<Char>(chained.size());
+        let hex = Text.toArray("0123456789abcdef");
+        for (c in chained.chars()) {
+            let n = Char.toNat32(c);
+            if (n < 0x20) {
+                buf.add('\\');
+                buf.add('u');
+                buf.add('0');
+                buf.add('0');
+                buf.add(hex[Nat32.toNat(n / 16)]);
+                buf.add(hex[Nat32.toNat(n % 16)]);
+            } else {
+                buf.add(c);
+            };
+        };
+        Text.fromIter(buf.vals())
+    };
+
     /// Converts serialized Candid blob to JSON text
     public func toText(blob : Blob, keys : [Text], options : ?CandidType.Options) : Result<Text, Text> {
         let decoded_res = Candid.decode(blob, keys, options);
         let #ok(candid) = decoded_res else return Utils.send_error(decoded_res);
 
-        let json_res = fromCandid(candid[0]);
+        let skip_null_fields = switch (options) {
+            case (?opts) opts.skip_null_fields;
+            case null false;
+        };
+
+        let json_res = fromCandidWith(candid[0], skip_null_fields);
         let #ok(json) = json_res else return Utils.send_error(json_res);
         #ok(json);
     };
 
-    /// Convert a Candid value to JSON text
-    public func fromCandid(candid : Candid) : Result<Text, Text> {
-        let res = candidToJSON(candid);
+    /// Convert a Candid value to JSON text (default: keep null fields).
+    public func fromCandid(candid : Candid) : Result<Text, Text> =
+        fromCandidWith(candid, false);
+
+    /// Convert a Candid value to JSON text with explicit null-skip behaviour.
+    public func fromCandidWith(candid : Candid, skip_null_fields : Bool) : Result<Text, Text> {
+        let res = candidToJSON(candid, skip_null_fields);
         let #ok(json) = res else return Utils.send_error(res);
 
         #ok(JSON.show(json));
     };
 
-    func candidToJSON(candid : Candid) : Result<JSON, Text> {
+    func candidToJSON(candid : Candid, skip_null_fields : Bool) : Result<JSON, Text> {
         let json : JSON = switch (candid) {
             case (#Null) #Null;
             case (#Bool(n)) #Boolean(n);
-            case (#Text(n)) #String(Text.replace(n, #text("\""), ("\\\"")));
+            case (#Text(n)) #String(escapeJSONString(n));
 
             case (#Int(n)) #Number(n);
             case (#Int8(n)) #Number(IntX.from8ToInt(n));
@@ -56,7 +99,7 @@ module {
             case (#Option(val)) {
                 let res = switch (val) {
                     case (#Null) return #ok(#Null);
-                    case (v) candidToJSON(v);
+                    case (v) candidToJSON(v, skip_null_fields);
                 };
 
                 let #ok(optional_val) = res else return Utils.send_error(res);
@@ -66,7 +109,7 @@ module {
                 let newArr = Buffer.Buffer<JSON>(arr.size());
 
                 for (item in arr.vals()) {
-                    let res = candidToJSON(item);
+                    let res = candidToJSON(item, skip_null_fields);
                     let #ok(json) = res else return Utils.send_error(res);
                     newArr.add(json);
                 };
@@ -78,17 +121,23 @@ module {
                 let newRecords = Buffer.Buffer<(Text, JSON)>(records.size());
 
                 for ((key, val) in records.vals()) {
-                    let res = candidToJSON(val);
+                    let res = candidToJSON(val, skip_null_fields);
                     let #ok(json) = res else return Utils.send_error(res);
-                    newRecords.add((key, json));
+                    // With `skip_null_fields`, entries whose value serialised
+                    // to JSON `null` are treated as "field absent" — matches
+                    // how external HTTP APIs read optional fields.
+                    switch (skip_null_fields, json) {
+                        case (true, #Null) ();
+                        case _ newRecords.add((key, json));
+                    };
                 };
 
                 #Object(Buffer.toArray(newRecords));
             };
 
             case (#Variant(variant)) {
                 let (key, val) = variant;
-                let res = candidToJSON(val);
+                let res = candidToJSON(val, skip_null_fields);
                 let #ok(json_val) = res else return Utils.send_error(res);
 
                 #Object([("#" # key, json_val)]);

src/JSON/lib.mo

@@ -13,7 +13,7 @@ module {
 
     public let { fromText; toCandid } = FromText;
 
-    public let { toText; fromCandid } = ToText;
+    public let { toText; fromCandid; fromCandidWith } = ToText;
 
     public let concatKeys = Utils.concatKeys;
 };

src/UrlEncoded/ToText.mo

@@ -24,11 +24,20 @@ module {
     public func toText(blob : Blob, keys : [Text], options : ?CandidType.Options) : Result<Text, Text> {
         let res = Candid.decode(blob, keys, options);
         let #ok(candid) = res else return Utils.send_error(res);
-        fromCandid(candid[0]);
+
+        let skip_null_fields = switch (options) {
+            case (?opts) opts.skip_null_fields;
+            case null false;
+        };
+        fromCandidWith(candid[0], skip_null_fields);
     };
 
-    /// Convert a Candid Record to a URL-Encoded string.
-    public func fromCandid(candid : Candid) : Result<Text, Text> {
+    /// Convert a Candid Record to a URL-Encoded string (default: keep null fields as `key=null`).
+    public func fromCandid(candid : Candid) : Result<Text, Text> =
+        fromCandidWith(candid, false);
+
+    /// Same as [fromCandid] but with explicit null-skip behaviour.
+    public func fromCandidWith(candid : Candid, skip_null_fields : Bool) : Result<Text, Text> {
 
         let records = switch (candid) {
             case (#Record(records) or #Map(records)) records;
@@ -39,7 +48,7 @@ module {
         let pairsOrder = Buffer.Buffer<Text>(16);
 
         for ((key, value) in records.vals()) {
-            toKeyValuePairs(pairsMap, pairsOrder, key, value);
+            toKeyValuePairs(pairsMap, pairsOrder, key, value, skip_null_fields);
         };
 
         var url_encoding = "";
@@ -65,6 +74,7 @@ module {
         pairsOrder : Buffer.Buffer<Text>,
         storedKey : Text,
         candid : Candid,
+        skip_null_fields : Bool,
     ) {
         func set(key : Text, value : Text) {
             if (Map.get(pairsMap, Map.thash, key) == null) {
@@ -76,26 +86,26 @@ module {
             case (#Array(arr)) {
                 for ((i, value) in itertools.enumerate(arr.vals())) {
                     let array_key = storedKey # "[" # Nat.toText(i) # "]";
-                    toKeyValuePairs(pairsMap, pairsOrder, array_key, value);
+                    toKeyValuePairs(pairsMap, pairsOrder, array_key, value, skip_null_fields);
                 };
             };
 
             case (#Record(records) or #Map(records)) {
                 for ((key, value) in records.vals()) {
                     let record_key = storedKey # "[" # key # "]";
-                    toKeyValuePairs(pairsMap, pairsOrder, record_key, value);
+                    toKeyValuePairs(pairsMap, pairsOrder, record_key, value, skip_null_fields);
                 };
             };
 
             case (#Variant(key, val)) {
                 let variant_key = storedKey # "#" # key;
-                toKeyValuePairs(pairsMap, pairsOrder, variant_key, val);
+                toKeyValuePairs(pairsMap, pairsOrder, variant_key, val, skip_null_fields);
             };
 
             // TODO: convert blob to hex
             // case (#Blob(blob)) set(storedKey, "todo: Blob.toText(blob)");
 
-            case (#Option(p)) toKeyValuePairs(pairsMap, pairsOrder, storedKey, p);
+            case (#Option(p)) toKeyValuePairs(pairsMap, pairsOrder, storedKey, p, skip_null_fields);
             case (#Text(t)) set(storedKey, t);
             case (#Principal(p)) set(storedKey, Principal.toText(p));
 
@@ -112,7 +122,9 @@ module {
             case (#Int64(n)) set(storedKey, U.stripStart(debug_show (n), #char '+'));
 
             case (#Float(n)) set(storedKey, Float.toText(n));
-            case (#Null) set(storedKey, "null");
+            // With `skip_null_fields`, omit the pair entirely rather than
+            // emitting `key=null` — matches the JSON/CBOR encoders.
+            case (#Null) if (not skip_null_fields) set(storedKey, "null");
             case (#Empty) set(storedKey, "");
 
             case (#Bool(b)) set(storedKey, debug_show (b));