merged networknt/light-4j#503 from master to 1.6.x

cherry-pick 26382993

Feat/#502 consul integrate test (#503)

* Fixed the scope caching error

* Added dependence of container test into pom

* Moved the test cases to server module

* Enhanced the registry test with testContainer.

* Migrate consul container test to integration folder

* Reverted unnecessary change

* Changed name

* Disable the integration test for mvn clean install

* Removed unused line

* -now integration test will copy the config under its own integration folder to the target folder
-CURD some config files due to fix test cases (those were not configured correctly in previous versions)

* -move integration plugins to integration-test profile
-test configs will be generated either from integration folder or under test folder depends on the profile, cannot include both because file with same name will have only one copy under config directory in phase "generate-test-sources".
-when run mvn clean install will as usual, when run mvn clean install -P integration-test will execute integration tests

* fixes the typo

# Conflicts:
#	pom.xml

codegen-cli/src/main/java/com/networknt/codegen/Cli.java

@@ -3,19 +3,17 @@
 import com.beust.jcommander.JCommander;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
-import com.fasterxml.jackson.databind.ObjectMapper;
 import com.fizzed.rocker.runtime.RockerRuntime;
 import com.jsoniter.JsonIterator;
 import com.jsoniter.any.Any;
+import com.networknt.codegen.rest.YAMLFileParameterizer;
 
-import java.io.BufferedInputStream;
-import java.io.ByteArrayOutputStream;
-import java.io.IOException;
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.util.Set;
+import static java.io.File.separator;
 
 /**
  * Created by steve on 24/04/17.
@@ -40,10 +38,13 @@ public class Cli {
 
     @Parameter(names={"--help", "-h"}, help = true)
     private boolean help;
-    
+
     @Parameter(names={"--reload", "-r"}, description = "Specifies whether rocker hot-reloading should be enabled or not. Default is false."+
-    "If this is set to true, the file 'rocker-compiler.conf' must be available in the classpath.")
-    private boolean reload;    
+            "If this is set to true, the file 'rocker-compiler.conf' must be available in the classpath.")
+    private boolean reload;
+
+    @Parameter(names={"--parameterize", "-p"}, description = "The location of configuration files that need to be parameterized.")
+    String parameterizationDir;
 
     public static void main(String ... argv) throws Exception {
         try {
@@ -65,7 +66,7 @@ public void run(JCommander jCommander) throws Exception {
             jCommander.usage();
             return;
         }
-        
+
         RockerRuntime.getInstance().setReloading(reload);
 
         System.out.printf("%s %s %s %s\n", framework, model, config, output);
@@ -101,6 +102,9 @@ public void run(JCommander jCommander) throws Exception {
                     anyConfig = JsonIterator.deserialize(Files.readAllBytes(Paths.get(config)));
                 }
             }
+            if(parameterizationDir != null) {
+                YAMLFileParameterizer.rewriteAll(parameterizationDir, output + separator + YAMLFileParameterizer.DEFAULT_DEST_DIR, anyConfig.asMap().get(YAMLFileParameterizer.GENERATE_ENV_VARS).asMap());
+            }
             generator.generate(output, anyModel, anyConfig);
             System.out.println("A project has been generated successfully in " + output + " folder. Have fun!!!");
         } else {
@@ -112,3 +116,4 @@ public void run(JCommander jCommander) throws Exception {
     }
 
 }
+

codegen-core/src/main/java/com/networknt/codegen/Generator.java

@@ -1,11 +1,7 @@
 package com.networknt.codegen;
 
-import com.fizzed.rocker.runtime.ArrayOfByteArraysOutput;
-import com.fizzed.rocker.runtime.DefaultRockerModel;
-import com.jsoniter.any.Any;
-import org.apache.commons.io.IOUtils;
-
 import java.io.File;
+import static java.io.File.separator;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
@@ -14,13 +10,14 @@
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.nio.file.StandardCopyOption;
-import java.nio.file.attribute.PosixFilePermission;
 import java.util.HashMap;
-import java.util.HashSet;
 import java.util.Map;
-import java.util.Set;
 
-import static java.io.File.separator;
+import com.fizzed.rocker.runtime.ArrayOfByteArraysOutput;
+import com.fizzed.rocker.runtime.DefaultRockerModel;
+import com.jsoniter.any.Any;
+
+import org.apache.commons.io.IOUtils;
 
 /**
  * This is the interface that every generator needs to implement. There are three methods
@@ -165,4 +162,13 @@ default boolean checkExist(String folder, String path, String filename) throws I
         return Files.exists(Paths.get(absPath));
     }
 
+    static void copyFile(final InputStream is, final java.nio.file.Path folder) throws IOException {
+        java.nio.file.Path parent = folder.getParent();
+        if (!Files.isDirectory(parent)) {
+            Files.createDirectories(parent);
+        }
+
+        Files.copy(is, folder, StandardCopyOption.REPLACE_EXISTING);
+    }
+
 }

codegen-web/pom.xml

@@ -82,6 +82,14 @@
             <groupId>com.networknt</groupId>
             <artifactId>resource</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>health</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.networknt</groupId>
+            <artifactId>info</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.networknt</groupId>
             <artifactId>json-schema-validator</artifactId>

codegen-web/src/main/java/com/networknt/codegen/handler/CodegenMultipleHandler.java

@@ -10,7 +10,6 @@
 import com.networknt.rpc.Handler;
 import com.networknt.rpc.router.JsonHandler;
 import com.networknt.rpc.router.ServiceHandler;
-import com.networknt.status.Status;
 import com.networknt.utility.HashUtil;
 import com.networknt.utility.NioUtils;
 import io.undertow.server.HttpServerExchange;
@@ -26,7 +25,6 @@
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.security.InvalidParameterException;
 import java.util.*;
 
 import static java.io.File.separator;

codegen-web/src/main/java/com/networknt/codegen/handler/CodegenSingleHandler.java

@@ -11,7 +11,6 @@
 import com.networknt.rpc.Handler;
 import com.networknt.rpc.router.JsonHandler;
 import com.networknt.rpc.router.ServiceHandler;
-import com.networknt.status.Status;
 import com.networknt.utility.HashUtil;
 import com.networknt.utility.NioUtils;
 import io.undertow.server.HttpServerExchange;

codegen-web/src/main/resources/config/handler.yml

@@ -0,0 +1,86 @@
+# Handler middleware chain configuration
+---
+enabled: true
+
+#------------------------------------------------------------------------------
+# Support individual handler chains for each separate endpoint. It allows framework
+# handlers like health check, server info to bypass majority of the middleware handlers
+# and allows mixing multiple frameworks like OpenAPI and GraphQL in the same instance.
+#
+# handlers  --  list of handlers to be used across chains in this microservice
+#               including the routing handlers for ALL endpoints
+#           --  format: fully qualified handler class name@optional:given name
+# chains    --  allows forming of [1..N] chains, which could be wholly or
+#               used to form handler chains for each endpoint
+#               ex.: default chain below, reused partially across multiple endpoints
+# paths     --  list all the paths to be used for routing within the microservice
+#           ----  path: the URI for the endpoint (ex.: path: '/v1/pets')
+#           ----  method: the operation in use (ex.: 'post')
+#           ----  exec: handlers to be executed -- this element forms the list and
+#                       the order of execution for the handlers
+#
+# IMPORTANT NOTES:
+# - to avoid executing a handler, it has to be removed/commented out in the chain
+#   or change the enabled:boolean to false for a middleware handler configuration.
+# - all handlers, routing handler included, are to be listed in the execution chain
+# - for consistency, give a name to each handler; it is easier to refer to a name
+#   vs a fully qualified class name and is more elegant
+# - you can list in chains the fully qualified handler class names, and avoid using the
+#   handlers element altogether
+#------------------------------------------------------------------------------
+handlers:
+  # Light-framework cross-cutting concerns implemented in the microservice
+  - com.networknt.exception.ExceptionHandler@exception
+  - com.networknt.metrics.MetricsHandler@metrics
+  - com.networknt.traceability.TraceabilityHandler@traceability
+  - com.networknt.correlation.CorrelationHandler@correlation
+  # Cors handler to handler post/put pre-flight
+  - com.networknt.cors.CorsHttpHandler@cors
+  # - com.networknt.openapi.OpenApiHandler@specification
+  # - com.networknt.openapi.JwtVerifyHandler@security
+  # - com.networknt.body.BodyHandler@body
+  # - com.networknt.audit.AuditHandler@audit
+  # - com.networknt.sanitizer.SanitizerHandler@sanitizer
+  # - com.networknt.openapi.ValidatorHandler@validator
+  # Header middleware to manipulate request and/or response headers before or after downstream server
+  # - com.networknt.header.HeaderHandler@header
+  # Direct requests to named services based on the request path
+  # - com.networknt.router.middleware.PathPrefixServiceHandler@path
+  # - com.networknt.router.RouterHandler@router
+  # - com.networknt.resource.PathResourceHandler@resource
+  - com.networknt.rpc.router.JsonHandler@json
+  # Customer business domain specific cross-cutting concerns handlers
+  # - com.example.validator.CustomizedValidator@custvalidator
+  # Framework endpoint handlers
+  - com.networknt.health.HealthGetHandler@health
+  - com.networknt.info.ServerInfoGetHandler@info
+  # - com.networknt.metrics.prometheus.PrometheusGetHandler@getprometheus
+
+chains:
+  default:
+    - exception
+    - metrics
+    - traceability
+    - correlation
+    - json
+
+paths:
+  - path: '/codegen'
+    method: 'POST'
+    exec:
+      - default
+  - path: '/codegen'
+    method: 'GET'
+    exec:
+      - default
+
+  - path: '/health/com.networknt.codegen-web-2.0.0'
+    method: 'get'
+    exec:
+      - health
+
+  # In most case, the /server/info endpoint shouldn't be exposed. If it is, then it must be protected by OAuth 2.0 or Basic Auth
+  - path: '/server/info'
+    method: 'get'
+    exec:
+      - info

codegen-web/src/main/resources/config/rpc-router.yml

@@ -0,0 +1,4 @@
+---
+description: RPC router description
+handlerPackage: 'com.networknt.codegen'
+jsonPath: /codegen

codegen-web/src/main/resources/config/server.yml

@@ -29,7 +29,7 @@ enableTwoWayTls: false
 truststoreName: server.truststore
 
 # Unique service identifier. Used in service registration and discovery etc.
-serviceId: com.networknt.example-graphql-subscription-1.0.0
+serviceId: com.networknt.codegen-web-1.6.0
 
 # Flag to enable service registration. Only be true if running as standalone Java jar.
 enableRegistry: false

codegen-web/src/main/resources/config/service.yml

@@ -11,20 +11,7 @@ singletons:
 # - com.networknt.server.ShutdownHookProvider:
   # - com.networknt.server.Test1ShutdownHook
   # - com.networknt.server.Test2ShutdownHook
-# MiddlewareHandler implementations
-- com.networknt.handler.MiddlewareHandler:
-  # Exception Global exception handler that needs to be called first.
-  - com.networknt.exception.ExceptionHandler
-  # Metrics In order to calculate response time accurately, this needs to be the second.
-  - com.networknt.metrics.MetricsHandler
-  # Traceability Put traceabilityId into response header from request header if it exists
-  - com.networknt.traceability.TraceabilityHandler
-  # Correlation Create correlationId if it doesn't exist in the request header and put it into the request header
-  - com.networknt.correlation.CorrelationHandler
-  # Cors handler to handler post/put pre-flight
-  - com.networknt.cors.CorsHttpHandler
-  # SimpleAudit Log important info about the request into audit log
-  - com.networknt.audit.AuditHandler
+
 # Generator interface implementations
 - com.networknt.codegen.Generator:
   - com.networknt.codegen.rest.OpenApiGenerator

codegen-web/src/main/resources/schema.json

@@ -53,9 +53,12 @@
               },
               "framework": {
                 "type": "string"
+              },
+              "release": {
+                "type": "string"
               }
             },
-            "required": ["configType", "modelType", "framework"]
+            "required": ["configType", "modelType", "framework", "release"]
           },
           "maxItems": 5,
           "minItems": 1
@@ -91,9 +94,12 @@
         },
         "framework": {
           "type" : "string"
+        },
+        "release": {
+          "type" : "string"
         }
       },
-      "required": ["configType", "modelType", "framework"]
+      "required": ["configType", "modelType", "framework", "release"]
     },
     "scope": "codegen.w"
   }

codegen-web/src/test/java/com/networknt/codegen/handler/FrameworkListHandlerTest.java

@@ -56,7 +56,7 @@ public void testGenerator() throws ClientException, ApiException, UnsupportedEnc
             connection.getIoThread().execute(new Runnable() {
                 @Override
                 public void run() {
-                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                     request.getRequestHeaders().put(Headers.HOST, "localhost");
                     request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");
                     request.getRequestHeaders().put(Headers.TRANSFER_ENCODING, "chunked");

codegen-web/src/test/java/com/networknt/codegen/handler/GeneratorServiceHandlerTest.java

@@ -5,7 +5,6 @@
 import com.networknt.client.Http2Client;
 import com.networknt.exception.ApiException;
 import com.networknt.exception.ClientException;
-import io.undertow.UndertowOptions;
 import io.undertow.client.ClientConnection;
 import io.undertow.client.ClientRequest;
 import io.undertow.client.ClientResponse;
@@ -20,7 +19,6 @@
 import org.xnio.OptionMap;
 
 import java.io.IOException;
-import java.io.UnsupportedEncodingException;
 import java.net.URI;
 import java.util.HashMap;
 import java.util.List;
@@ -80,7 +78,7 @@ public void testMissingGeneratorItem() throws ClientException {
         }
         try {
             connection.getIoThread().execute(() -> {
-                final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
 
                 request.getRequestHeaders().put(Headers.HOST, "localhost");
                 request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
@@ -133,7 +131,7 @@ public void testInvalidFrameworkMultiple() throws ClientException, IOException {
         }
         try {
             connection.getIoThread().execute(() -> {
-                final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                 request.getRequestHeaders().put(Headers.HOST, "localhost");
                 request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                 request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");
@@ -185,7 +183,7 @@ public void testInvalidFrameworkSingle() throws ClientException, IOException {
         }
         try {
             connection.getIoThread().execute(() -> {
-                final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                 request.getRequestHeaders().put(Headers.HOST, "localhost");
                 request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                 request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");
@@ -234,7 +232,7 @@ public void testGeneratorMultipleText() throws ClientException, ApiException, IO
             connection.getIoThread().execute(new Runnable() {
                 @Override
                 public void run() {
-                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                     request.getRequestHeaders().put(Headers.HOST, "localhost");
                     request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                     request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");
@@ -278,7 +276,7 @@ public void testGeneratorSingleText() throws ClientException, ApiException, IOEx
             connection.getIoThread().execute(new Runnable() {
                 @Override
                 public void run() {
-                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                     request.getRequestHeaders().put(Headers.HOST, "localhost");
                     request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                     request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");
@@ -322,7 +320,7 @@ public void testGeneratorMultipleAllUrl() throws ClientException, ApiException,
             connection.getIoThread().execute(new Runnable() {
                 @Override
                 public void run() {
-                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                     request.getRequestHeaders().put(Headers.HOST, "localhost");
                     request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                     request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");
@@ -366,7 +364,7 @@ public void testGeneratorSingleModelUrl() throws ClientException, ApiException,
             connection.getIoThread().execute(new Runnable() {
                 @Override
                 public void run() {
-                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                     request.getRequestHeaders().put(Headers.HOST, "localhost");
                     request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                     request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");

codegen-web/src/test/java/com/networknt/codegen/handler/SchemaGetHandlerTest.java

@@ -55,7 +55,7 @@ public void testGetSchema() throws ClientException, ApiException, UnsupportedEnc
             connection.getIoThread().execute(new Runnable() {
                 @Override
                 public void run() {
-                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/api/json");
+                    final ClientRequest request = new ClientRequest().setMethod(Methods.POST).setPath("/codegen");
                     request.getRequestHeaders().put(Headers.HOST, "localhost");
                     request.getRequestHeaders().put(Headers.AUTHORIZATION, auth);
                     request.getRequestHeaders().put(Headers.CONTENT_TYPE, "application/json");