Merge pull request #146 from schemacrawler/allow-children

Allow Graphviz diagram output

Author: sualeh

.github/workflows/docker-image-early-release.yml

@@ -110,7 +110,7 @@ jobs:
         git push --delete origin early-access-release || true
         git tag -a early-access-release -m "SchemaCrawler AI Early Access Release"
         git push --follow-tags origin early-access-release
-        git show early-access-release
+        git show --no-patch early-access-release
 
 # BUILD AND PUBLISH DOCKER IMAGE
     - id: setup-qemu

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/DescribeRoutinesFunctionParameters.java

@@ -25,9 +25,8 @@ public record DescribeRoutinesFunctionParameters(
             Name of database routine (stored procedure or function) to describe.
             May be specified as a regular expression matching the fully qualified
             stored procedure or function names (including the schema).
-            Use an empty string if all routines are requested.
-            If not specified, all routines are returned, but the results
-            could be large.
+            Try not to match all routines, but instead use a regular expression
+            to match a subset, since otherwise results may be large.
             """)
         @JsonProperty(required = false)
         String routineName,

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/DescribeTablesFunctionExecutor.java

@@ -54,9 +54,11 @@ protected SchemaCrawlerOptions createSchemaCrawlerOptions() {
             .includeSequences(new ExcludeAll())
             .includeRoutines(new ExcludeAll())
             .includeTables(new IncludeAll());
+
     final InclusionRule grepTablesPattern = makeInclusionRule(commandOptions.tableName());
     final GrepOptionsBuilder grepOptionsBuilder =
         GrepOptionsBuilder.builder().includeGreppedTables(grepTablesPattern);
+
     return SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
         .withLimitOptions(limitOptionsBuilder.toOptions())
         .withGrepOptions(grepOptionsBuilder.toOptions());

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/DescribeTablesFunctionParameters.java

@@ -25,9 +25,9 @@ public record DescribeTablesFunctionParameters(
             Name of database table or view to describe.
             May be specified as a regular expression, matching the fully qualified
             table name (including the schema).
-            Use an empty string if all tables are requested.
-            If not specified, all tables will be returned, but the results
-            could be large.
+            Try not to match all tables, but instead use a regular expression
+            to match a subset or match a single table, since otherwise results may
+            be large.
             """)
         @JsonProperty(required = false)
         String tableName,

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/DiagramFunctionExecutor.java

@@ -10,6 +10,10 @@
 
 import java.nio.file.Path;
 import schemacrawler.inclusionrule.InclusionRule;
+import schemacrawler.schemacrawler.FilterOptionsBuilder;
+import schemacrawler.schemacrawler.GrepOptionsBuilder;
+import schemacrawler.schemacrawler.SchemaCrawlerOptions;
+import schemacrawler.schemacrawler.SchemaCrawlerOptionsBuilder;
 import schemacrawler.tools.ai.functions.DiagramFunctionParameters.DiagramType;
 import schemacrawler.tools.ai.tools.FunctionReturn;
 import schemacrawler.tools.ai.tools.base.AbstractExecutableFunctionExecutor;
@@ -27,20 +31,45 @@ protected DiagramFunctionExecutor(final PropertyName functionName) {
 
   @Override
   public FunctionReturn call() {
-    final Config additionalConfig = createAdditionalConfig();
-    final InclusionRule grepTablesInclusionRule = makeInclusionRule(commandOptions.tableName());
+    final DiagramType diagramType = commandOptions.diagramType();
+    final Config additionalConfig = createAdditionalConfig(diagramType);
     final ExecutionParameters executionParameters =
-        new ExecutionParameters("script", additionalConfig, grepTablesInclusionRule, "text");
+        new ExecutionParameters(
+            diagramType.getCommand(), additionalConfig, diagramType.getOutputFormatValue());
     final Path outputFilePath = execute(executionParameters);
     return returnText(outputFilePath);
   }
 
-  private Config createAdditionalConfig() {
+  @Override
+  protected SchemaCrawlerOptions createSchemaCrawlerOptions() {
+    final FilterOptionsBuilder filterOptionsBuilder = FilterOptionsBuilder.builder();
+    if (commandOptions.includeChildTables()) {
+      filterOptionsBuilder.childTableFilterDepth(1);
+    }
+    if (commandOptions.includeReferencedTables()) {
+      filterOptionsBuilder.parentTableFilterDepth(1);
+    }
+
+    final InclusionRule grepTablesInclusionRule = makeInclusionRule(commandOptions.tableName());
+    final GrepOptionsBuilder grepOptionsBuilder =
+        GrepOptionsBuilder.builder().includeGreppedTables(grepTablesInclusionRule);
+
+    final SchemaCrawlerOptions schemaCrawlerOptions =
+        SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
+            .withFilterOptions(filterOptionsBuilder.build())
+            .withGrepOptions(grepOptionsBuilder.toOptions());
+    return schemaCrawlerOptions;
+  }
+
+  private Config createAdditionalConfig(final DiagramType diagramType) {
     final Config additionalConfig = ConfigUtility.newConfig();
+    if (diagramType == null || diagramType == DiagramType.GRAPHVIZ) {
+      return additionalConfig;
+    }
 
-    final DiagramType diagramType = commandOptions.diagramType();
     additionalConfig.put("script-language", "python");
     additionalConfig.put("script", diagramType.script());
+
     return additionalConfig;
   }
 }

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/DiagramFunctionParameters.java

@@ -22,39 +22,69 @@ public record DiagramFunctionParameters(
             Name of database table or view to describe.
             May be specified as a regular expression, matching the fully qualified
             table name (including the schema).
-            Use an empty string if all tables are requested.
-            If not specified, all tables will be returned, but the results
-            could be large.
+            Try not to match all tables, but instead use a regular expression
+            to match a subset or match a single table, since otherwise results may
+            be large.
             """)
         @JsonProperty(required = false)
         String tableName,
     @JsonPropertyDescription(
             """
-            Indicates database schema diagram format - Graphviz, PlantUML, Mermaid
-            or DBML from dbdiagram.io.
+            If true, also include child (or dependent) tables for the selected tables.
+            """)
+        @JsonProperty(required = false, defaultValue = "false")
+        boolean includeChildTables,
+    @JsonPropertyDescription(
+            """
+            If true, also include tables that are referenced by the selected tables.
+            (These are sometimes known as parent tables.)
+            """)
+        @JsonProperty(required = false, defaultValue = "false")
+        boolean includeReferencedTables,
+    @JsonPropertyDescription(
+            """
+            Indicates database schema diagram format - Graphviz DOT format, PlantUML,
+            Mermaid or DBML from dbdiagram.io.
             """)
         @JsonProperty(required = true)
         DiagramType diagramType)
     implements FunctionParameters {
 
   public enum DiagramType {
-    PLANTUML("/scripts/plantuml.py", "https://editor.plantuml.com/"),
-    MERMAID("/scripts/mermaid.py", "https://mermaid.live/"),
-    DBML("/scripts/dbml.py", "https://dbdiagram.io/d"),
+    PLANTUML("script", "text", "/scripts/plantuml.py", "https://editor.plantuml.com/"),
+    MERMAID("script", "text", "/scripts/mermaid.py", "https://mermaid.live/"),
+    DBML("script", "text", "/scripts/dbml.py", "https://dbdiagram.io/d"),
+    GRAPHVIZ("schema", "scdot", "", "https://dreampuf.github.io/GraphvizOnline/"),
     ;
 
+    private final String command;
+    private final String outputFormatValue;
     private final String script;
     private final String onlineEditorUrl;
 
-    DiagramType(final String script, final String url) {
+    DiagramType(
+        final String command,
+        final String outputFormatValue,
+        final String script,
+        final String url) {
+      this.command = command;
+      this.outputFormatValue = outputFormatValue;
       this.script = script;
       onlineEditorUrl = url;
     }
 
+    public String getCommand() {
+      return command;
+    }
+
     public String getOnlineEditorUrl() {
       return onlineEditorUrl;
     }
 
+    public String getOutputFormatValue() {
+      return outputFormatValue;
+    }
+
     public String script() {
       return script;
     }

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/LintFunctionExecutor.java

@@ -10,6 +10,9 @@
 
 import java.nio.file.Path;
 import schemacrawler.inclusionrule.InclusionRule;
+import schemacrawler.schemacrawler.GrepOptionsBuilder;
+import schemacrawler.schemacrawler.SchemaCrawlerOptions;
+import schemacrawler.schemacrawler.SchemaCrawlerOptionsBuilder;
 import schemacrawler.tools.ai.tools.FunctionReturn;
 import schemacrawler.tools.ai.tools.base.AbstractExecutableFunctionExecutor;
 import schemacrawler.tools.ai.tools.base.ExecutionParameters;
@@ -26,10 +29,19 @@ protected LintFunctionExecutor(final PropertyName functionName) {
   @Override
   public FunctionReturn call() {
     final String outputFormat = LintReportOutputFormat.json.name();
-    final InclusionRule grepTablesInclusionRule = makeInclusionRule(commandOptions.tableName());
-    final ExecutionParameters executionParameters =
-        new ExecutionParameters("lint", grepTablesInclusionRule, outputFormat);
+    final ExecutionParameters executionParameters = new ExecutionParameters("lint", outputFormat);
     final Path outputFilePath = execute(executionParameters);
     return returnJson(outputFilePath);
   }
+
+  @Override
+  protected SchemaCrawlerOptions createSchemaCrawlerOptions() {
+    final InclusionRule grepTablesInclusionRule = makeInclusionRule(commandOptions.tableName());
+    final GrepOptionsBuilder grepOptionsBuilder =
+        GrepOptionsBuilder.builder().includeGreppedTables(grepTablesInclusionRule);
+    final SchemaCrawlerOptions schemaCrawlerOptions =
+        SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
+            .withGrepOptions(grepOptionsBuilder.toOptions());
+    return schemaCrawlerOptions;
+  }
 }

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/functions/TableSampleFunctionExecutor.java

@@ -10,6 +10,9 @@
 
 import java.nio.file.Path;
 import schemacrawler.inclusionrule.InclusionRule;
+import schemacrawler.schemacrawler.GrepOptionsBuilder;
+import schemacrawler.schemacrawler.SchemaCrawlerOptions;
+import schemacrawler.schemacrawler.SchemaCrawlerOptionsBuilder;
 import schemacrawler.tools.ai.tools.FunctionReturn;
 import schemacrawler.tools.ai.tools.base.AbstractExecutableFunctionExecutor;
 import schemacrawler.tools.ai.tools.base.ExecutionParameters;
@@ -24,10 +27,19 @@ protected TableSampleFunctionExecutor(final PropertyName functionName) {
 
   @Override
   public FunctionReturn call() {
-    final InclusionRule grepTablesInclusionRule = makeInclusionRule(commandOptions.tableName());
-    final ExecutionParameters executionParameters =
-        new ExecutionParameters("tablesample", grepTablesInclusionRule, "json");
+    final ExecutionParameters executionParameters = new ExecutionParameters("tablesample", "json");
     final Path outputFilePath = execute(executionParameters);
     return returnJson(outputFilePath);
   }
+
+  @Override
+  protected SchemaCrawlerOptions createSchemaCrawlerOptions() {
+    final InclusionRule grepTablesInclusionRule = makeInclusionRule(commandOptions.tableName());
+    final GrepOptionsBuilder grepOptionsBuilder =
+        GrepOptionsBuilder.builder().includeGreppedTables(grepTablesInclusionRule);
+    final SchemaCrawlerOptions schemaCrawlerOptions =
+        SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
+            .withGrepOptions(grepOptionsBuilder.toOptions());
+    return schemaCrawlerOptions;
+  }
 }

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/tools/base/AbstractExecutableFunctionExecutor.java

@@ -21,8 +21,6 @@
 import java.util.logging.Level;
 import java.util.logging.Logger;
 import schemacrawler.inclusionrule.ExcludeAll;
-import schemacrawler.inclusionrule.InclusionRule;
-import schemacrawler.schemacrawler.GrepOptionsBuilder;
 import schemacrawler.schemacrawler.LimitOptionsBuilder;
 import schemacrawler.schemacrawler.SchemaCrawlerOptions;
 import schemacrawler.schemacrawler.SchemaCrawlerOptionsBuilder;
@@ -44,6 +42,7 @@
 import us.fatehi.utility.datasource.DatabaseConnectionSource;
 import us.fatehi.utility.datasource.DatabaseConnectionSources;
 import us.fatehi.utility.property.PropertyName;
+import us.fatehi.utility.string.StringFormat;
 
 public abstract class AbstractExecutableFunctionExecutor<P extends FunctionParameters>
     extends AbstractFunctionExecutor<P> {
@@ -65,8 +64,7 @@ protected final Path execute(final ExecutionParameters executionParameters) {
     requireNonNull(executionParameters, "No execution parameters provided");
 
     // Crate SchemaCrawler options
-    final SchemaCrawlerOptions options =
-        createSchemaCrawlerOptions(executionParameters.grepTablesInclusionRule());
+    final SchemaCrawlerOptions options = adjustSchemaCrawlerOptions();
 
     // Re-filter catalog
     MetaDataUtility.reduceCatalog(catalog, options);
@@ -84,8 +82,9 @@ protected final Path execute(final ExecutionParameters executionParameters) {
     final Config config = createAdditionalConfig(executionParameters.additionalConfig());
 
     // Create executable
-    final SchemaCrawlerExecutable executable =
-        new SchemaCrawlerExecutable(executionParameters.command());
+    final String command = executionParameters.command();
+    LOGGER.log(Level.INFO, new StringFormat("Executing SchemaCrawler command <%s>", command));
+    final SchemaCrawlerExecutable executable = new SchemaCrawlerExecutable(command);
     executable.setSchemaCrawlerOptions(options);
     executable.setCatalog(catalog);
     if (connection != null) {
@@ -143,6 +142,26 @@ protected final FunctionReturn returnText(final Path outputFilePath) {
     }
   }
 
+  private final SchemaCrawlerOptions adjustSchemaCrawlerOptions() {
+
+    final SchemaCrawlerOptions baseOptions = createSchemaCrawlerOptions();
+    final LimitOptionsBuilder limitOptionsBuilder =
+        LimitOptionsBuilder.builder()
+            .includeSynonyms(new ExcludeAll())
+            .includeSequences(new ExcludeAll())
+            .includeRoutines(new ExcludeAll());
+    SchemaCrawlerOptions schemaCrawlerOptions =
+        SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
+            .withLimitOptions(limitOptionsBuilder.toOptions());
+    if (baseOptions != null) {
+      schemaCrawlerOptions =
+          schemaCrawlerOptions
+              .withFilterOptions(baseOptions.filterOptions())
+              .withGrepOptions(baseOptions.grepOptions());
+    }
+    return schemaCrawlerOptions;
+  }
+
   private Config createAdditionalConfig(final Config additionalConfig) {
     final Config config = SchemaTextOptionsBuilder.builder().noInfo().toConfig();
     config.merge(additionalConfig);
@@ -166,20 +185,6 @@ private OutputOptions createOutputOptions(final String outputFormatValue, final
     return outputOptions;
   }
 
-  private final SchemaCrawlerOptions createSchemaCrawlerOptions(
-      final InclusionRule grepTablesInclusionRule) {
-    final LimitOptionsBuilder limitOptionsBuilder =
-        LimitOptionsBuilder.builder()
-            .includeSynonyms(new ExcludeAll())
-            .includeSequences(new ExcludeAll())
-            .includeRoutines(new ExcludeAll());
-    final GrepOptionsBuilder grepOptionsBuilder =
-        GrepOptionsBuilder.builder().includeGreppedTables(grepTablesInclusionRule);
-    return SchemaCrawlerOptionsBuilder.newSchemaCrawlerOptions()
-        .withLimitOptions(limitOptionsBuilder.toOptions())
-        .withGrepOptions(grepOptionsBuilder.toOptions());
-  }
-
   private void deleteTempFile(final Path outputFilePath) {
     if (outputFilePath == null) {
       return;

schemacrawler-ai-core/src/main/java/schemacrawler/tools/ai/tools/base/AbstractFunctionExecutor.java

@@ -15,6 +15,7 @@
 import schemacrawler.inclusionrule.IncludeAll;
 import schemacrawler.inclusionrule.InclusionRule;
 import schemacrawler.inclusionrule.RegularExpressionInclusionRule;
+import schemacrawler.schemacrawler.SchemaCrawlerOptions;
 import schemacrawler.tools.ai.tools.FunctionExecutor;
 import schemacrawler.tools.ai.tools.FunctionParameters;
 import schemacrawler.tools.ai.tools.FunctionReturn;
@@ -33,6 +34,8 @@ public final String toString() {
     return command.getName();
   }
 
+  protected abstract SchemaCrawlerOptions createSchemaCrawlerOptions();
+
   protected InclusionRule makeInclusionRule(final String objectName) {
     final InclusionRule inclusionRule;
     if (isBlank(objectName)) {