From 43ec45981cbb3f08b19d882e86f068e7eebc6c66 Mon Sep 17 00:00:00 2001 From: Elad Ben-Israel Date: Mon, 3 Feb 2025 19:02:49 +0200 Subject: [PATCH] fix: structs can't parse json with null values (#7258) Fixes #7257 by accepting `null` for any optional field in structs and converting `null` to `undefined` during parsing from JSON to structs. ## Checklist - [x] Title matches [Winglang's style guide](https://www.winglang.io/contributing/start-here/pull_requests#how-are-pull-request-titles-formatted) - [x] Description explains motivation and solution - [x] Tests added (always) - [x] Docs updated (only required for features) - [x] Added `pr/e2e-full` label if this feature requires end-to-end testing *By submitting this pull request, I confirm that my contribution is made under the terms of the [Wing Cloud Contribution License](https://github.com/winglang/wing/blob/main/CONTRIBUTION_LICENSE.md)*. --------- Signed-off-by: monada-bot[bot] Co-authored-by: monada-bot[bot] --- docs/api/05-language-reference.md | 3 + .../01-start-here/05-development.md | 59 ++++++++++++------- packages/@winglang/sdk/src/std/json_schema.ts | 19 +++++- .../wingc/src/json_schema_generator.rs | 5 +- tests/valid/optionals.test.w | 34 +++++++++++ .../valid/optionals.test.w_compile_tf-aws.md | 14 +++++ .../valid/optionals.test.w_test_sim.md | 2 + .../parameters.test.w_compile_tf-aws.md | 2 +- .../struct_from_json.test.w_compile_tf-aws.md | 8 +-- 9 files changed, 119 insertions(+), 27 deletions(-) diff --git a/docs/api/05-language-reference.md b/docs/api/05-language-reference.md index f280f6fa600..b256debab64 100644 --- a/docs/api/05-language-reference.md +++ b/docs/api/05-language-reference.md @@ -418,6 +418,9 @@ let x = Contact.fromJson(p, unsafe: true); assert(x.last.len > 0); // RUNTIME ERROR ``` +> NOTE: `fromJson()` and `parseJson()` treats `null` values as `undefined` and therefore can be +> assigned to optional fields. + ##### 1.1.4.8 Serialization The `Json.stringify(j: Json): str` static method can be used to serialize a `Json` as a string diff --git a/docs/contributing/01-start-here/05-development.md b/docs/contributing/01-start-here/05-development.md index b4f85dc880f..9dc5f57fa62 100644 --- a/docs/contributing/01-start-here/05-development.md +++ b/docs/contributing/01-start-here/05-development.md @@ -98,39 +98,50 @@ pnpm build It will compile, lint, test and package all modules. -## ๐Ÿ  What's the recommended development workflow? -:::info -When testing your changes to Wing, locally it may be helpful to be able to easily invoke your local version of the Wing CLI. -In which case adding a shell alias may be helpful for instance on Linux and Mac you could add: - -`alias mywing=//packages/winglang/bin/wing` to your shell's rc file. -::: +## What's the recommended development workflow? ๐Ÿ  -The `pnpm wing` command can be executed from the root of the repository in order to build and run the -compiler, SDK (standard library) and the Wing CLI. Turbo is configured to make sure only the changed components are built +The `pnpm wing` command can be executed from the *root of the repository* in order to build +everything and run Wing CLI. Turbo is configured to make sure only the changed components are built every time. + +:::info To get full diagnostics, use these exports: ```sh export NODE_OPTIONS=--stack-trace-limit=100 export RUST_BACKTRACE=full ``` +::: -Or if you just want to compile your changes and run a local version of the Wing CLI: +Now, you can edit a source file anywhere across the stack and run the compiler with arguments. +For example: ```sh -turbo compile -F winglang +pnpm wing -- test tests/valid/captures.test.w ``` -Now, you can edit a source file anywhere across the stack and run the compiler with arguments. -For example: +This command runs the full Wing CLI with the given arguments. Turbo will ensure the CLI build is updated. + +:::info +When testing your changes to Wing locally it may be helpful to be able to easily invoke your local version of the Wing CLI. + +First, you need to compile changes: ```sh -pnpm wing -- test examples/tests/valid/captures.test.w +npx turbo compile -F winglang ``` -This command runs the full Wing CLI with the given arguments. Turbo will ensure the CLI build is updated. +Then, run the Wing CLI binary directly: + +```sh +./packages/winglang/bin/wing +``` + +Pro tip: create a shell alias: + +`alias mywing=//packages/winglang/bin/wing` to your shell's rc file. +::: ## How is the repository structured? @@ -154,7 +165,7 @@ If you wish to install it manually, you may do so by running `scripts/setup_wasi ::: -## ๐Ÿงช How do I run tests? +## How do I run tests? ๐Ÿงช End-to-end tests are hosted under `tools/hangar`. To get started, first ensure you can [build wing](#-how-do-i-build-wing). @@ -167,6 +178,14 @@ turbo wing:e2e (This is a helpful shortcut for `turbo test -F hangar`) +To run a single test, use the `wing test` from the root and reference the test file name: + +For example: + +```sh +pnpm wing -- test tests/valid/optionals.test.w +``` + ### Test Meta-Comments In your wing files in `examples/tests/valid`, you can add a specially formatted comment to add additional information for hangar. @@ -186,7 +205,7 @@ Currently, the only supported meta-comment for regular tests is `skipPlatforms`. This will skip the test on the given platforms when when running on CI. The current supported platforms are `win32`, `darwin`, and `linux`. This is useful if, for example, the test requires docker. In our CI only linux supports docker. -### Benchmarks +## Performance Benchmarks Benchmark files are located in `examples/tests/valid/benchmarks`. To run the benchmarks, run the following command from anywhere in the monorepo: @@ -256,7 +275,7 @@ highlight queries, run: turbo playground -F @winglang/tree-sitter-wing ``` -## ๐Ÿ”จ How do I build the VSCode extension? +## How do I build the VSCode extension? ๐Ÿ”จ The VSCode extension is located in `packages/vscode-wing`. Most of the "logic" is in the language server, which is located in the Wing CLI at `packages/winglang/src/commands/lsp.ts`. @@ -277,7 +296,7 @@ To modify the package.json, make sure to edit `.projenrc.ts` and rebuild. Tip: if you want to print debug messages in your code while developing, you should use Rust's `dbg!` macro, instead of `print!` or `println!`. -## ๐Ÿงน How do I lint my code? +## How do I lint my code? ๐Ÿงน To lint Rust code, you can run the `lint` target on the `wingc` or `wingii` projects: @@ -294,7 +313,7 @@ Lastly you can show linting errors in your IDE by enabling the following setting "rust-analyzer.check.command": "clippy", ``` -## ๐Ÿ How do I add a quickstart template to the `wing` CLI? +## How do I add a quickstart template to the `wing` CLI? ๐Ÿ Adding a new template is straightforward! diff --git a/packages/@winglang/sdk/src/std/json_schema.ts b/packages/@winglang/sdk/src/std/json_schema.ts index a22f3003503..376fffe0315 100644 --- a/packages/@winglang/sdk/src/std/json_schema.ts +++ b/packages/@winglang/sdk/src/std/json_schema.ts @@ -77,7 +77,10 @@ export class JsonSchema { const fields = extractFieldsFromSchema(this._rawSchema); // Filter rawParameters based on the schema const filteredParameters = filterParametersBySchema(fields, obj); - return filteredParameters; + + // Remove all `null` values (recursively) + const cleanedParameters = removeNullValues(filteredParameters); + return cleanedParameters; } /** @internal */ @@ -103,3 +106,17 @@ export class JsonSchema { return JsonSchema._toInflightType(this._rawSchema); } } + +function removeNullValues(obj: any): any { + if (typeof obj === "object" && !Array.isArray(obj)) { + const result: any = {}; + for (const [key, value] of Object.entries(obj)) { + if (value !== null) { + result[key] = removeNullValues(value); + } + } + return result; + } + + return obj; +} diff --git a/packages/@winglang/wingc/src/json_schema_generator.rs b/packages/@winglang/wingc/src/json_schema_generator.rs index 417d5c9d2b5..452e21309f9 100644 --- a/packages/@winglang/wingc/src/json_schema_generator.rs +++ b/packages/@winglang/wingc/src/json_schema_generator.rs @@ -99,7 +99,10 @@ impl JsonSchemaGenerator { code.append("}"); code.to_string() } - Type::Optional(ref t) => self.get_struct_schema_field(t, docs), + Type::Optional(ref t) => format!( + "{{oneOf:[{{type:\"null\"}},{}]}}", + self.get_struct_schema_field(t, docs) + ), Type::Json(_) => match docs { Some(docs) => format!( "{{type:[\"object\",\"string\",\"boolean\",\"number\",\"array\"],description:\"{}\"}}", diff --git a/tests/valid/optionals.test.w b/tests/valid/optionals.test.w index bd5e538dc07..d2adb1bd811 100644 --- a/tests/valid/optionals.test.w +++ b/tests/valid/optionals.test.w @@ -256,3 +256,37 @@ assert(maybeX! == 0); let maybeY: str? = ""; assert(maybeY! == ""); + +// ------------------------------------------------------------------------------------------------ +// verify that `null` is treated as undefined/nil +// https://github.com/winglang/wing/issues/7257 + +struct S1 { + x: str?; +} + +log(S1.schema().asStr()); + +let s9 = S1.parseJson("\{\"x\": null}"); +assert(s9.x == nil); + +struct S2 { + y: S1?; +} + +log(S2.schema().asStr()); +let s10 = S2.parseJson("\{\"y\": null}"); +assert(s10.y == nil); + +let s11 = S2.parseJson("\{\"y\": \{\"x\": null\}}"); +assert(s11.y?.x == nil); + +struct S3 { + arr: Array?; + map: Map?; +} + +let s12 = S3.parseJson("\{\"arr\": null,\"map\": null}"); +assert(s12.arr == nil); +assert(s12.map == nil); +// ------------------------------------------------------------------------------------------------ diff --git a/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_compile_tf-aws.md b/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_compile_tf-aws.md index 80b3d9a2636..2218743f21f 100644 --- a/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_compile_tf-aws.md +++ b/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_compile_tf-aws.md @@ -159,6 +159,9 @@ class $Root extends $stdlib.std.Resource { let $preflightTypesMap = {}; const cloud = $stdlib.cloud; const Person = $stdlib.std.Struct._createJsonSchema({$id:"/Person",type:"object",properties:{age:{type:"number"},name:{type:"string"},},required:["age","name",],description:""}); + const S1 = $stdlib.std.Struct._createJsonSchema({$id:"/S1",type:"object",properties:{x:{oneOf:[{type:"null"},{type:"string"}]},},required:[],description:""}); + const S2 = $stdlib.std.Struct._createJsonSchema({$id:"/S2",type:"object",properties:{y:{oneOf:[{type:"null"},{type:"object",properties:{x:{oneOf:[{type:"null"},{type:"string"}]},},required:[],description:""}]},},required:[],description:""}); + const S3 = $stdlib.std.Struct._createJsonSchema({$id:"/S3",type:"object",properties:{arr:{oneOf:[{type:"null"},{type:"array",items:{type:"string"}}]},map:{oneOf:[{type:"null"},{type:"object",patternProperties: {".*":{type:"string"}},}]},},required:[],description:""}); $helpers.nodeof(this).root.$preflightTypesMap = $preflightTypesMap; class Super extends $stdlib.std.Resource { constructor($scope, $id, ) { @@ -506,6 +509,17 @@ class $Root extends $stdlib.std.Resource { $helpers.assert($helpers.eq($helpers.unwrap(maybeX), 0), "maybeX! == 0"); const maybeY = ""; $helpers.assert($helpers.eq($helpers.unwrap(maybeY), ""), "maybeY! == \"\""); + console.log(($macros.__Struct_schema(false, S1, ).asStr())); + const s9 = $macros.__Struct_parseJson(false, S1, "{\"x\": null}"); + $helpers.assert($helpers.eq(s9.x, undefined), "s9.x == nil"); + console.log(($macros.__Struct_schema(false, S2, ).asStr())); + const s10 = $macros.__Struct_parseJson(false, S2, "{\"y\": null}"); + $helpers.assert($helpers.eq(s10.y, undefined), "s10.y == nil"); + const s11 = $macros.__Struct_parseJson(false, S2, "{\"y\": {\"x\": null\}}"); + $helpers.assert($helpers.eq(s11.y?.x, undefined), "s11.y?.x == nil"); + const s12 = $macros.__Struct_parseJson(false, S3, "{\"arr\": null,\"map\": null}"); + $helpers.assert($helpers.eq(s12.arr, undefined), "s12.arr == nil"); + $helpers.assert($helpers.eq(s12.map, undefined), "s12.map == nil"); } } const $APP = $PlatformManager.createApp({ outdir: $outdir, name: "optionals.test", rootConstruct: $Root, isTestEnvironment: $wing_is_test, entrypointDir: process.env['WING_SOURCE_DIR'], rootId: process.env['WING_ROOT_ID'] }); diff --git a/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_test_sim.md b/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_test_sim.md index 1507aeb02b1..32c5055e087 100644 --- a/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_test_sim.md +++ b/tools/hangar/__snapshots__/test_corpus/valid/optionals.test.w_test_sim.md @@ -2,6 +2,8 @@ ## stdout.log ```log +{"$id":"/S1","type":"object","properties":{"x":{"oneOf":[{"type":"null"},{"type":"string"}]}},"required":[],"description":""} +{"$id":"/S2","type":"object","properties":{"y":{"oneOf":[{"type":"null"},{"type":"object","properties":{"x":{"oneOf":[{"type":"null"},{"type":"string"}]}},"required":[],"description":""}]}},"required":[],"description":""} pass โ”€ optionals.test.wsim ยป root/Default/test:t Tests 1 passed (1) diff --git a/tools/hangar/__snapshots__/test_corpus/valid/parameters/simple/parameters.test.w_compile_tf-aws.md b/tools/hangar/__snapshots__/test_corpus/valid/parameters/simple/parameters.test.w_compile_tf-aws.md index b952266e5fc..ee2f3d859f2 100644 --- a/tools/hangar/__snapshots__/test_corpus/valid/parameters/simple/parameters.test.w_compile_tf-aws.md +++ b/tools/hangar/__snapshots__/test_corpus/valid/parameters/simple/parameters.test.w_compile_tf-aws.md @@ -36,7 +36,7 @@ class $Root extends $stdlib.std.Resource { super($scope, $id); $helpers.nodeof(this).root.$preflightTypesMap = { }; let $preflightTypesMap = {}; - const MyParams = $stdlib.std.Struct._createJsonSchema({$id:"/MyParams",type:"object",properties:{foo:{type:"string"},meaningOfLife:{type:"number"},},required:["meaningOfLife",],description:""}); + const MyParams = $stdlib.std.Struct._createJsonSchema({$id:"/MyParams",type:"object",properties:{foo:{oneOf:[{type:"null"},{type:"string"}]},meaningOfLife:{type:"number"},},required:["meaningOfLife",],description:""}); $helpers.nodeof(this).root.$preflightTypesMap = $preflightTypesMap; const myParams = $macros.__Struct_fromJson(false, MyParams, ($helpers.nodeof(this).app.parameters.read({ schema: $macros.__Struct_schema(false, MyParams, ) }))); { diff --git a/tools/hangar/__snapshots__/test_corpus/valid/struct_from_json.test.w_compile_tf-aws.md b/tools/hangar/__snapshots__/test_corpus/valid/struct_from_json.test.w_compile_tf-aws.md index 5eca7b2987b..add73c798bc 100644 --- a/tools/hangar/__snapshots__/test_corpus/valid/struct_from_json.test.w_compile_tf-aws.md +++ b/tools/hangar/__snapshots__/test_corpus/valid/struct_from_json.test.w_compile_tf-aws.md @@ -199,11 +199,11 @@ class $Root extends $stdlib.std.Resource { const otherExternalStructs = $helpers.bringJs(`${__dirname}/preflight.structs2-2.cjs`, $preflightTypesMap); const Bar = $stdlib.std.Struct._createJsonSchema({$id:"/Bar",type:"object",properties:{b:{type:"number"},f:{type:"string"},},required:["b","f",],description:""}); const Foo = $stdlib.std.Struct._createJsonSchema({$id:"/Foo",type:"object",properties:{f:{type:"string"},},required:["f",],description:""}); - const Foosible = $stdlib.std.Struct._createJsonSchema({$id:"/Foosible",type:"object",properties:{f:{type:"string"},},required:[],description:""}); + const Foosible = $stdlib.std.Struct._createJsonSchema({$id:"/Foosible",type:"object",properties:{f:{oneOf:[{type:"null"},{type:"string"}]},},required:[],description:""}); const MyStruct = $stdlib.std.Struct._createJsonSchema({$id:"/MyStruct",type:"object",properties:{color:{type:"string",enum:["red", "green", "blue"],description:"color docs\n@example Color.red"},m1:{type:"object",properties:{val:{type:"number",description:"val docs"},},required:["val",],description:"m1 docs"},m2:{type:"object",properties:{val:{type:"string"},},required:["val",],description:"m2 docs"},},required:["color","m1","m2",],description:"MyStruct docs\n@foo bar"}); - const Student = $stdlib.std.Struct._createJsonSchema({$id:"/Student",type:"object",properties:{additionalData:{type:["object","string","boolean","number","array"]},advisor:{type:"object",properties:{dob:{type:"object",properties:{day:{type:"number"},month:{type:"number"},year:{type:"number"},},required:["day","month","year",],description:""},employeeID:{type:"string"},firstName:{type:"string"},lastName:{type:"string"},},required:["dob","employeeID","firstName","lastName",],description:""},coursesTaken:{type:"array",items:{type:"object",properties:{course:{type:"object",properties:{credits:{type:"number"},name:{type:"string"},},required:["credits","name",],description:""},dateTaken:{type:"object",properties:{day:{type:"number"},month:{type:"number"},year:{type:"number"},},required:["day","month","year",],description:""},grade:{type:"string"},},required:["course","dateTaken","grade",],description:""}},dob:{type:"object",properties:{day:{type:"number"},month:{type:"number"},year:{type:"number"},},required:["day","month","year",],description:""},enrolled:{type:"boolean"},enrolledCourses:{type:"array",uniqueItems:true,items:{type:"object",properties:{credits:{type:"number"},name:{type:"string"},},required:["credits","name",],description:""}},firstName:{type:"string"},lastName:{type:"string"},schoolId:{type:"string"},},required:["dob","enrolled","firstName","lastName","schoolId",],description:""}); - const cloud_ApiResponse = $stdlib.std.Struct._createJsonSchema({$id:"/ApiResponse",type:"object",properties:{body:{type:"string",description:"The response\'s body.\n@default - no body\n@stability experimental"},headers:{type:"object",patternProperties: {".*":{type:"string"}},description:"The response\'s headers.\n@default {}\n@stability experimental",},status:{type:"number",description:"The response\'s status code.\n@default 200\n@stability experimental"},},required:[],description:"Shape of a response from a inflight handler.\n@stability experimental"}); - const cloud_CounterProps = $stdlib.std.Struct._createJsonSchema({$id:"/CounterProps",type:"object",properties:{initial:{type:"number",description:"The initial value of the counter.\n@default 0\n@stability experimental"},},required:[],description:"Options for `Counter`.\n@stability experimental"}); + const Student = $stdlib.std.Struct._createJsonSchema({$id:"/Student",type:"object",properties:{additionalData:{oneOf:[{type:"null"},{type:["object","string","boolean","number","array"]}]},advisor:{oneOf:[{type:"null"},{type:"object",properties:{dob:{type:"object",properties:{day:{type:"number"},month:{type:"number"},year:{type:"number"},},required:["day","month","year",],description:""},employeeID:{type:"string"},firstName:{type:"string"},lastName:{type:"string"},},required:["dob","employeeID","firstName","lastName",],description:""}]},coursesTaken:{oneOf:[{type:"null"},{type:"array",items:{type:"object",properties:{course:{type:"object",properties:{credits:{type:"number"},name:{type:"string"},},required:["credits","name",],description:""},dateTaken:{type:"object",properties:{day:{type:"number"},month:{type:"number"},year:{type:"number"},},required:["day","month","year",],description:""},grade:{type:"string"},},required:["course","dateTaken","grade",],description:""}}]},dob:{type:"object",properties:{day:{type:"number"},month:{type:"number"},year:{type:"number"},},required:["day","month","year",],description:""},enrolled:{type:"boolean"},enrolledCourses:{oneOf:[{type:"null"},{type:"array",uniqueItems:true,items:{type:"object",properties:{credits:{type:"number"},name:{type:"string"},},required:["credits","name",],description:""}}]},firstName:{type:"string"},lastName:{type:"string"},schoolId:{type:"string"},},required:["dob","enrolled","firstName","lastName","schoolId",],description:""}); + const cloud_ApiResponse = $stdlib.std.Struct._createJsonSchema({$id:"/ApiResponse",type:"object",properties:{body:{oneOf:[{type:"null"},{type:"string",description:"The response\'s body.\n@default - no body\n@stability experimental"}]},headers:{oneOf:[{type:"null"},{type:"object",patternProperties: {".*":{type:"string"}},description:"The response\'s headers.\n@default {}\n@stability experimental",}]},status:{oneOf:[{type:"null"},{type:"number",description:"The response\'s status code.\n@default 200\n@stability experimental"}]},},required:[],description:"Shape of a response from a inflight handler.\n@stability experimental"}); + const cloud_CounterProps = $stdlib.std.Struct._createJsonSchema({$id:"/CounterProps",type:"object",properties:{initial:{oneOf:[{type:"null"},{type:"number",description:"The initial value of the counter.\n@default 0\n@stability experimental"}]},},required:[],description:"Options for `Counter`.\n@stability experimental"}); const externalStructs_MyOtherStruct = $stdlib.std.Struct._createJsonSchema({$id:"/MyOtherStruct",type:"object",properties:{data:{type:"object",properties:{val:{type:"number",description:"val docs"},},required:["val",],description:"MyStruct docs in subdir"},},required:["data",],description:""}); $helpers.nodeof(this).root.$preflightTypesMap = $preflightTypesMap; const Color =