Add in ability to define additional connection options when creating …

…plan

app/src/main/resources/ui/configuration-data.js

@@ -1252,3 +1252,22 @@ dataSourcePropertiesMap.set("slack", {
         }
     }
 });
+
+export let subDataSourceConfigMap = new Map();
+subDataSourceConfigMap.set("http", {
+    method: {
+        displayName: "Method",
+        default: "N/A",
+        type: "text",
+        help: "HTTP method.",
+        choice: ["N/A", "GET", "POST", "PUT", "DELETE", "PATCH", "CONNECT", "OPTIONS", "TRACE", "HEAD"],
+        override: "true"
+    },
+    endpoint: {
+        displayName: "Endpoint",
+        default: "",
+        type: "text",
+        help: "Endpoint pathway (i.e. '/my-path/data').",
+        override: "true"
+    }
+});

app/src/main/resources/ui/helper-foreign-keys.js

@@ -7,13 +7,16 @@ Ability to choose task name and columns. Define custom relationships.
  */
 import {
     addAccordionCloseButton,
+    addConnectionOverrideOptions,
     createAccordionItem,
     createButton,
     createCloseButton,
     createFieldValidationCheck,
     createFormFloating,
     createInput,
     createSelect,
+    createTooltip,
+    getOverrideConnectionOptionsAsMap,
     wait
 } from "./shared.js";
 
@@ -47,7 +50,8 @@ async function createForeignKeyLinksFromPlan(newForeignKey, foreignKey, linkType
     for (const fkLink of Array.from(foreignKey[`${linkType}Links`])) {
         let newForeignKeyLink = await createForeignKeyInput(numForeignKeysLinks, `foreign-key-${linkType}-link`);
         foreignKeyLinkSources.insertBefore(newForeignKeyLink, foreignKeyLinkSources.lastChild);
-        $(newForeignKeyLink).find(`select.foreign-key-${linkType}-link`).selectpicker("val", fkLink.taskName);
+        $(newForeignKeyLink).find(`select.foreign-key-${linkType}-link`).selectpicker("val", fkLink.taskName)[0].dispatchEvent(new Event("change"));
+        ;
         $(newForeignKeyLink).find(`input.foreign-key-${linkType}-link`).val(fkLink.columns)[0].dispatchEvent(new Event("input"));
     }
 }
@@ -61,7 +65,8 @@ export async function createForeignKeysFromPlan(respJson) {
             foreignKeysAccordion.append(newForeignKey);
 
             if (foreignKey.source) {
-                $(newForeignKey).find("select.foreign-key-source").selectpicker("val", foreignKey.source.taskName);
+                $(newForeignKey).find("select.foreign-key-source").selectpicker("val", foreignKey.source.taskName)[0].dispatchEvent(new Event("change"));
+                ;
                 $(newForeignKey).find("input.foreign-key-source").val(foreignKey.source.columns)[0].dispatchEvent(new Event("input"));
             }
 
@@ -76,22 +81,23 @@ export async function createForeignKeysFromPlan(respJson) {
 }
 
 function getForeignKeyLinksToArray(fkContainer, className) {
-    let fkGenerationLinks = $(fkContainer).find(className);
-    let fkGenerationLinkArray = [];
-    for (let fkLink of fkGenerationLinks) {
+    let mainContainer = $(fkContainer).find(className);
+    let fkLinks = $(mainContainer).find(".foreign-key-input-container");
+    let fkLinksArray = [];
+    for (let fkLink of fkLinks) {
         let fkLinkDetails = getForeignKeyDetail(fkLink);
-        fkGenerationLinkArray.push(fkLinkDetails);
+        fkLinksArray.push(fkLinkDetails);
     }
-    return fkGenerationLinkArray;
+    return fkLinksArray;
 }
 
 export function getForeignKeys() {
     let foreignKeyContainers = Array.from(document.querySelectorAll(".foreign-key-container").values());
     return foreignKeyContainers.map(fkContainer => {
         let fkSource = $(fkContainer).find(".foreign-key-main-source");
         let fkSourceDetails = getForeignKeyDetail(fkSource[0]);
-        let fkGenerationLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-generation-link-source");
-        let fkDeleteLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-delete-link-source");
+        let fkGenerationLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-generation-link-sources");
+        let fkDeleteLinkArray = getForeignKeyLinksToArray(fkContainer, ".foreign-key-delete-link-sources");
         return {source: fkSourceDetails, generationLinks: fkGenerationLinkArray, deleteLinks: fkDeleteLinkArray};
     });
 }
@@ -106,6 +112,10 @@ async function createForeignKeyLinks(index, linkType) {
     let addLinkForeignKeyButton = createButton(`add-foreign-key-${linkType}-link-btn-${index}`, "add-link", "btn btn-secondary", "+ Link");
     addLinkForeignKeyButton.addEventListener("click", async function () {
         numForeignKeysLinks += 1;
+        if (linkSourceForeignKeys.childElementCount > 1) {
+            let divider = document.createElement("hr");
+            linkSourceForeignKeys.insertBefore(divider, addLinkForeignKeyButton);
+        }
         let newForeignKeyLink = await createForeignKeyInput(numForeignKeysLinks, `foreign-key-${linkType}-link`);
         linkSourceForeignKeys.insertBefore(newForeignKeyLink, addLinkForeignKeyButton);
     });
@@ -161,6 +171,8 @@ async function updateForeignKeyTasks(taskNameSelect) {
 }
 
 async function createForeignKeyInput(index, name) {
+    let foreignKeyContainer = document.createElement("div");
+    foreignKeyContainer.setAttribute("class", "foreign-key-input-container m-1");
     let foreignKey = document.createElement("div");
     foreignKey.setAttribute("class", `row m-1 align-items-center ${name}-source`);
     // input is task name -> column(s)
@@ -175,6 +187,22 @@ async function createForeignKeyInput(index, name) {
     let columnNameFloating = createFormFloating("Column(s)", columnNamesInput);
 
     foreignKey.append(taskNameCol, columnNameFloating);
+    //when task name is selected, offer input to define sub data source if not defined
+    //(i.e. schema and table for Postgres task with no schema and table defined, only offer table if schema is defined in data source)
+    //for a http data source, endpoint is not part of the data source
+    //same logic can be shared for data generation/validation to allow re-use of connection
+    let iconDiv = createTooltip();
+    let overrideOptionsContainer = document.createElement("div");
+    overrideOptionsContainer.setAttribute("class", "foreign-key-connection-container");
+    taskNameSelect.addEventListener("change", (event) => {
+        let taskName = event.target.value;
+        //get the corresponding task data source connection name
+        let taskNameInput = $(document).find(`input[class~=task-name-field]`).filter(function () {
+            return this.value === taskName
+        });
+        let connectionName = $(taskNameInput).closest("[class~=row]").find("select[class~=data-connection-name]").val();
+        addConnectionOverrideOptions(connectionName, iconDiv, overrideOptionsContainer, "foreign-key-connection-property", index);
+    });
     if (name === "foreign-key-generation-link" || name === "foreign-key-delete-link") {
         let closeButton = createCloseButton(foreignKey);
         foreignKey.append(closeButton);
@@ -188,11 +216,17 @@ async function createForeignKeyInput(index, name) {
         updateForeignKeyTasks(taskNameSelect);
     });
     await updateForeignKeyTasks(taskNameSelect);
-    return foreignKey;
+    foreignKeyContainer.append(foreignKey, overrideOptionsContainer);
+    return foreignKeyContainer;
 }
 
 function getForeignKeyDetail(element) {
     let taskName = $(element).find("select[aria-label=Task]").val();
     let columns = $(element).find("input[aria-label=Columns]").val();
-    return {taskName: taskName, columns: columns};
+    let baseForeignKey = {taskName: taskName, columns: columns};
+    let overrideConnectionOptions = getOverrideConnectionOptionsAsMap(element);
+    if (Object.keys(overrideConnectionOptions).length > 0) {
+        baseForeignKey["options"] = overrideConnectionOptions;
+    }
+    return baseForeignKey;
 }

app/src/main/resources/ui/index.js

@@ -203,16 +203,17 @@ async function createDataConnectionInput(index) {
     dataConnectionCol.setAttribute("class", "col");
     dataConnectionCol.append(dataConnectionSelect);
 
-    let iconDiv = createIconWithConnectionTooltip(dataConnectionSelect);
+    // provide opportunity to override non-connection options for metadata source (i.e. namespace, dataset)
+    let overrideOptionsContainer = document.createElement("div");
+    let iconDiv = createIconWithConnectionTooltip(dataConnectionSelect, overrideOptionsContainer, "data-source-property", index);
     let iconCol = document.createElement("div");
     iconCol.setAttribute("class", "col-md-auto");
     iconCol.append(iconDiv);
 
-    // let inputGroup = createInputGroup(dataConnectionSelect, iconDiv, "col");
-    // $(inputGroup).find(".input-group").addClass("align-items-center");
     baseTaskDiv.append(taskNameFormFloating, dataConnectionCol, iconCol);
+    let baseDivWithSelectOptions = await getDataConnectionsAndAddToSelect(dataConnectionSelect, baseTaskDiv, "dataSource");
 
-    return await getDataConnectionsAndAddToSelect(dataConnectionSelect, baseTaskDiv, "dataSource");
+    return [baseDivWithSelectOptions, overrideOptionsContainer];
 }
 
 /*
@@ -225,7 +226,7 @@ async function createDataSourceConfiguration(index, closeButton, divider) {
     let divContainer = document.createElement("div");
     divContainer.setAttribute("id", "data-source-config-container-" + index);
     divContainer.setAttribute("class", "data-source-config-container");
-    let dataConnectionFormFloating = await createDataConnectionInput(index);
+    let [dataConnectionFormFloating, overrideConnectionOptionsContainer] = await createDataConnectionInput(index);
     let dataConfigAccordion = document.createElement("div");
     dataConfigAccordion.setAttribute("class", "accordion mt-2");
     let dataGenConfigContainer = createDataConfigElement(index, "generation");
@@ -236,7 +237,7 @@ async function createDataSourceConfiguration(index, closeButton, divider) {
         divContainer.append(divider);
     }
     dataConnectionFormFloating.append(closeButton);
-    divContainer.append(dataConnectionFormFloating, dataConfigAccordion);
+    divContainer.append(dataConnectionFormFloating, overrideConnectionOptionsContainer, dataConfigAccordion);
     return divContainer;
 }
 
@@ -255,6 +256,10 @@ function createReportConfiguration() {
     }
 }
 
+function getOverrideConnectionOptions(dataSource, currentDataSource) {
+    currentDataSource["options"] = getOverrideConnectionOptionsAsMap(dataSource, currentDataSource);
+}
+
 createReportConfiguration();
 submitForm();
 savePlan();
@@ -274,6 +279,7 @@ function getPlanDetails(form) {
         getGeneration(dataSource, currentDataSource);
         getValidations(dataSource, currentDataSource);
         getRecordCount(dataSource, currentDataSource);
+        getOverrideConnectionOptions(dataSource, currentDataSource);
         allUserInputs.push(currentDataSource);
     }
 

app/src/main/resources/ui/shared.js

@@ -1,4 +1,9 @@
-import {dataTypeOptionsMap, validationTypeDisplayNameMap, validationTypeOptionsMap} from "./configuration-data.js";
+import {
+    dataSourcePropertiesMap,
+    dataTypeOptionsMap, subDataSourceConfigMap,
+    validationTypeDisplayNameMap,
+    validationTypeOptionsMap
+} from "./configuration-data.js";
 import {
     addColumnValidationBlock,
     incValidations,
@@ -715,7 +720,7 @@ export async function getDataConnectionsAndAddToSelect(dataConnectionSelect, bas
         });
 }
 
-export function createIconWithConnectionTooltip(dataConnectionSelect) {
+export function createTooltip() {
     let iconDiv = document.createElement("i");
     iconDiv.setAttribute("class", "bi bi-info-circle");
     iconDiv.setAttribute("data-bs-toggle", "tooltip");
@@ -724,34 +729,61 @@ export function createIconWithConnectionTooltip(dataConnectionSelect) {
     iconDiv.setAttribute("data-bs-html", "true");
     iconDiv.setAttribute("data-bs-title", "Connection options");
     new bootstrap.Tooltip(iconDiv);
-    // on select change, update icon title
-    dataConnectionSelect.addEventListener("change", (event) => {
-        let connectionName = event.target.value;
-        fetch(`http://localhost:9898/connection/${connectionName}`, {method: "GET"})
-            .then(r => {
-                if (r.ok) {
-                    return r.json();
-                } else {
-                    r.text().then(text => {
-                        createToast(`Get connection ${connectionName}`, `Failed to get connection ${connectionName}! Error: ${err}`, "fail");
-                        throw new Error(text);
-                    });
+    return iconDiv;
+}
+
+export function addConnectionOverrideOptions(connectionName, iconDiv, overrideOptionsContainer, propertyClass, index) {
+    fetch(`http://localhost:9898/connection/${connectionName}`, {method: "GET"})
+        .then(r => {
+            if (r.ok) {
+                return r.json();
+            } else {
+                r.text().then(text => {
+                    createToast(`Get connection ${connectionName}`, `Failed to get connection ${connectionName}! Error: ${err}`, "fail");
+                    throw new Error(text);
+                });
+            }
+        })
+        .then(respJson => {
+            if (respJson) {
+                let optionsToShow = {};
+                optionsToShow["type"] = respJson.type;
+                for (let [key, value] of Object.entries(respJson.options)) {
+                    if (key !== "user" && key !== "password") {
+                        optionsToShow[key] = value;
+                    }
                 }
-            })
-            .then(respJson => {
-                if (respJson) {
-                    let optionsToShow = {};
-                    optionsToShow["type"] = respJson.type;
-                    for (let [key, value] of Object.entries(respJson.options)) {
-                        if (key !== "user" && key !== "password") {
-                            optionsToShow[key] = value;
-                        }
+                let summary = Object.entries(optionsToShow).map(kv => `${kv[0]}: ${kv[1]}`).join("<br>");
+                iconDiv.setAttribute("data-bs-title", summary);
+                new bootstrap.Tooltip(iconDiv);
+
+                //allow overriding sub data source options if not defined in connection
+                //remove previous properties
+                overrideOptionsContainer.replaceChildren();
+                let dataSourceType = respJson.type;
+                let dataSourceProperties = dataSourcePropertiesMap.get(dataSourceType).properties;
+                let propertyEntries = Object.entries(dataSourceProperties);
+                let additionalProperties = subDataSourceConfigMap.has(respJson.type) ? subDataSourceConfigMap.get(respJson.type) : {};
+                Object.entries(additionalProperties).forEach(k => propertyEntries.push(k));
+
+                for (const [key, value] of propertyEntries) {
+                    let isConnectionOptionOverridable = value["override"] && value["override"] === "true";
+                    let isConnectionOptionEmptyOrMissing = !respJson.options[key] || respJson.options[key] === "";
+                    if (isConnectionOptionOverridable && isConnectionOptionEmptyOrMissing) {
+                        //add properties that can be overridden
+                        addNewDataTypeAttribute(key, value, `connection-config-${index}-${key}`, propertyClass, overrideOptionsContainer);
                     }
-                    let summary = Object.entries(optionsToShow).map(kv => `${kv[0]}: ${kv[1]}`).join("<br>");
-                    iconDiv.setAttribute("data-bs-title", summary);
-                    new bootstrap.Tooltip(iconDiv);
                 }
-            });
+            }
+        });
+}
+
+export function createIconWithConnectionTooltip(dataConnectionSelect, overrideOptionsContainer, propertyClass, index) {
+    let iconDiv = createTooltip();
+    // on select change, update icon title
+    dataConnectionSelect.addEventListener("change", (event) => {
+        let connectionName = event.target.value;
+        addConnectionOverrideOptions(connectionName, iconDiv, overrideOptionsContainer, propertyClass, index);
     });
     return iconDiv;
 }
@@ -915,6 +947,15 @@ export async function createNewField(index, type) {
     return accordionItem;
 }
 
+export function getOverrideConnectionOptionsAsMap(dataSource) {
+    return $(dataSource).find("[id^=connection-config-]").toArray()
+        .reduce(function (map, option) {
+            if (option.value !== "") {
+                map[option.getAttribute("aria-label")] = option.value;
+            }
+            return map;
+        }, {});
+}
 
 export const wait = function (ms = 1000) {
     return new Promise(resolve => {

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/model/JsonSupport.scala

@@ -31,7 +31,7 @@ trait JsonSupport extends SprayJsonSupport with DefaultJsonProtocol {
   implicit val waitRequestFormat: RootJsonFormat[WaitRequest] = jsonFormat1(WaitRequest.apply)
   implicit val validationItemRequestsFormat: RootJsonFormat[ValidationItemRequests] = rootFormat(lazyFormat(jsonFormat1(ValidationItemRequests.apply)))
   implicit val validationItemRequestFormat: RootJsonFormat[ValidationItemRequest] = rootFormat(lazyFormat(jsonFormat4(ValidationItemRequest.apply)))
-  implicit val foreignKeyItemRequestFormat: RootJsonFormat[ForeignKeyRequestItem] = jsonFormat2(ForeignKeyRequestItem.apply)
+  implicit val foreignKeyItemRequestFormat: RootJsonFormat[ForeignKeyRequestItem] = jsonFormat3(ForeignKeyRequestItem.apply)
   implicit val foreignKeyRequestFormat: RootJsonFormat[ForeignKeyRequest] = jsonFormat3(ForeignKeyRequest.apply)
   implicit val dataSourceRequestFormat: RootJsonFormat[DataSourceRequest] = jsonFormat7(DataSourceRequest.apply)
   implicit val configurationRequestFormat: RootJsonFormat[ConfigurationRequest] = jsonFormat6(ConfigurationRequest.apply)

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/model/models.scala

@@ -67,7 +67,7 @@ case class ForeignKeyRequest(
                               deleteLinks: List[ForeignKeyRequestItem] = List(),
                             )
 
-case class ForeignKeyRequestItem(taskName: String, columns: String)
+case class ForeignKeyRequestItem(taskName: String, columns: String, options: Option[Map[String, String]] = None)
 
 case class ConfigurationRequest(
                                  flag: Map[String, String] = Map(),

app/src/main/scala/io/github/datacatering/datacaterer/core/ui/plan/PlanRepository.scala

@@ -91,7 +91,8 @@ object PlanRepository extends JsonSupport {
     // get connection info
     val dataSourcesWithConnectionInfo = planRunRequest.dataSources.map(ds => {
       val connectionInfo = ConnectionRepository.getConnection(ds.name, false)
-      ds.copy(`type` = Some(connectionInfo.`type`), options = Some(connectionInfo.options))
+      val allConnectionOptions = connectionInfo.options ++ ds.options.getOrElse(Map())
+      ds.copy(`type` = Some(connectionInfo.`type`), options = Some(allConnectionOptions))
     })
     val planRunWithConnectionInfo = planRunRequest.copy(dataSources = dataSourcesWithConnectionInfo)
     // create new run id

gradle.properties

@@ -1,5 +1,5 @@
 groupId=io.github.data-catering
-version=0.9.1
+version=0.9.2
 
 scalaVersion=2.12
 scalaSpecificVersion=2.12.15