Add s3 support for generate CRUD (#6264)

* Add s3 support for generate CRUD
- Dropdown enhancement to open options on initial load
- Hide column selection option for s3

* Refactor the prop name

* Add useS3BucketList hook WIP

* Dropdown enchancement & Fix small issues

* Add fetch all sheets query

* Add Query to get all S3 buckets

* Fix dropdown open issue

* Remove defaultIsOpen prop from dropdown

* Resolve comments
- Remove debugger
- mockSheetUrl -> getSheetUrl

* Add S3 cypress test

* Fix cypress test yml config

* Fix generate page cypress test

Author: Rishabh Rathod

.github/workflows/client-test.yml

@@ -371,6 +371,8 @@ jobs:
           CYPRESS_TESTPASSWORD1: ${{ secrets.CYPRESS_TESTPASSWORD1 }}
           CYPRESS_TESTUSERNAME2: ${{ secrets.CYPRESS_TESTUSERNAME2 }}
           CYPRESS_TESTPASSWORD2: ${{ secrets.CYPRESS_TESTPASSWORD1 }}
+          CYPRESS_S3_ACCESS_KEY: ${{ secrets.CYPRESS_S3_ACCESS_KEY }}
+          CYPRESS_S3_SECRET_KEY: ${{ secrets.CYPRESS_S3_SECRET_KEY }}
           APPSMITH_DISABLE_TELEMETRY: true
           APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }}
           POSTGRES_PASSWORD: postgres

.github/workflows/external-client-test.yml

@@ -354,6 +354,8 @@ jobs:
           CYPRESS_TESTPASSWORD1: ${{ secrets.CYPRESS_TESTPASSWORD1 }}
           CYPRESS_TESTUSERNAME2: ${{ secrets.CYPRESS_TESTUSERNAME2 }}
           CYPRESS_TESTPASSWORD2: ${{ secrets.CYPRESS_TESTPASSWORD1 }}
+          CYPRESS_S3_ACCESS_KEY: ${{ secrets.CYPRESS_S3_ACCESS_KEY }}
+          CYPRESS_S3_SECRET_KEY: ${{ secrets.CYPRESS_S3_SECRET_KEY }}
           APPSMITH_DISABLE_TELEMETRY: true
           APPSMITH_GOOGLE_MAPS_API_KEY: ${{ secrets.APPSMITH_GOOGLE_MAPS_API_KEY }}
           POSTGRES_PASSWORD: postgres

app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/GenerateCRUD/GenerateCRUDPage_Spec.js

@@ -4,9 +4,15 @@ import homePage from "../../../../locators/HomePage.json";
 import datasource from "../../../../locators/DatasourcesEditor.json";
 
 describe("Generate New CRUD Page Inside from entity explorer", function() {
+  let datasourceName;
+
   before(() => {
     cy.startRoutesForDatasource();
     cy.createPostgresDatasource();
+
+    cy.get("@createDatasource").then((httpResponse) => {
+      datasourceName = httpResponse.response.body.data.name;
+    });
     // TODO
     // 1. Add INVALID credential for a datasource and test the invalid datasource structure flow.
     // 2. Add 2 supported datasource and 1 not supported datasource with a fixed name to search.
@@ -27,7 +33,7 @@ describe("Generate New CRUD Page Inside from entity explorer", function() {
     cy.get(generatePage.selectDatasourceDropdown).click();
 
     cy.get(generatePage.datasourceDropdownOption)
-      .first()
+      .contains(datasourceName)
       .click();
 
     cy.wait("@getDatasourceStructure").should(
@@ -81,7 +87,8 @@ describe("Generate New CRUD Page Inside from entity explorer", function() {
     cy.fillPostgresDatasourceForm();
 
     cy.generateUUID().then((UUID) => {
-      cy.renameDatasource(`PostgresSQL CRUD Demo ${UUID}`);
+      datasourceName = `PostgresSQL CRUD Demo ${UUID}`;
+      cy.renameDatasource(datasourceName);
     });
 
     cy.startRoutesForDatasource();
@@ -124,10 +131,16 @@ describe("Generate New CRUD Page Inside from entity explorer", function() {
     cy.get(pages.integrationActiveTab)
       .should("be.visible")
       .click({ force: true });
+    cy.wait(1000);
 
-    cy.get(generatePage.datasourceCardGeneratePageBtn)
-      .first()
-      .click();
+    cy.get(datasource.datasourceCard)
+      .contains(datasourceName)
+      .scrollIntoView()
+      .should("be.visible")
+      .closest(datasource.datasourceCard)
+      .within(() => {
+        cy.get(datasource.datasourceCardGeneratePageBtn).click();
+      });
 
     cy.wait("@getDatasourceStructure").should(
       "have.nested.property",

app/client/cypress/integration/Smoke_TestSuite/ClientSideTests/GenerateCRUD/S3_CRUDPage_Spec.js

@@ -0,0 +1,62 @@
+const pages = require("../../../../locators/Pages.json");
+const generatePage = require("../../../../locators/GeneratePage.json");
+
+describe("Generate New CRUD Page Inside from entity explorer", function() {
+  let datasourceName;
+  before(() => {
+    cy.startRoutesForDatasource();
+    cy.createAmazonS3Datasource();
+
+    cy.get("@createDatasource").then((httpResponse) => {
+      datasourceName = httpResponse.response.body.data.name;
+    });
+  });
+
+  it("Add new Page and generate CRUD template using existing supported datasource", function() {
+    cy.get(pages.AddPage)
+      .first()
+      .click();
+    cy.wait("@createPage").should(
+      "have.nested.property",
+      "response.body.responseMeta.status",
+      201,
+    );
+
+    cy.get(generatePage.generateCRUDPageActionCard).click();
+
+    cy.get(generatePage.selectDatasourceDropdown).click();
+
+    cy.get(generatePage.datasourceDropdownOption)
+      .contains(datasourceName)
+      .click();
+
+    // fetch bucket
+    cy.wait("@datasourceQuery").should(
+      "have.nested.property",
+      "response.body.responseMeta.status",
+      200,
+    );
+
+    cy.get(generatePage.selectTableDropdown).click();
+
+    cy.get(generatePage.dropdownOption)
+      .contains("assets-test.appsmith.com")
+      .scrollIntoView()
+      .should("be.visible")
+      .click();
+    //  skip optional search column selection.
+    cy.get(generatePage.generatePageFormSubmitBtn).click();
+
+    cy.wait("@replaceLayoutWithCRUDPage").should(
+      "have.nested.property",
+      "response.body.responseMeta.status",
+      201,
+    );
+    cy.wait("@getActions");
+    cy.wait("@postExecute").should(
+      "have.nested.property",
+      "response.body.responseMeta.status",
+      200,
+    );
+  });
+});

app/client/cypress/locators/DatasourcesEditor.json

@@ -16,6 +16,7 @@
   "activeDatasourceList": ".t--active-datasource-list",
   "datasourceCard": ".t--datasource",
   "datasourceCardMenu": ".t--datasource-menu-option",
+  "datasourceCardGeneratePageBtn": ".t--generate-template",
   "datasourceMenuOptionEdit": "t--datasource-option-edit",
   "datasourceMenuOptionDelete":"t--datasource-option-delete",
   "editDatasource": ".t--edit-datasource",
@@ -30,7 +31,7 @@
   "MsSQL": ".t--plugin-name:contains('MsSQL')",
   "Firestore": ".t--plugin-name:contains('Firestore')",
   "Redshift": ".t--plugin-name:contains('Redshift')",
-  "AmazonS3": ".t--plugin-name:contains('Amazon S3')",
+  "AmazonS3": ".t--plugin-name:contains('S3')",
   "authType": "[data-cy=authType]",
   "OAuth2": "//div[contains(@class,'option') and text()='OAuth 2.0']",
   "accessTokenUrl": "[data-cy='authentication.accessTokenUrl'] input",

app/client/cypress/locators/GeneratePage.json

@@ -6,6 +6,5 @@
   "selectTableDropdown":"[data-cy=t--table-dropdown]",
   "dropdownOption": ".bp3-popover-content .t--dropdown-option",
   "selectSearchColumnDropdown":"[data-cy=t--searchColumn-dropdown]",
-  "generatePageFormSubmitBtn":"[data-cy=t--generate-page-form-submit]",
-  "datasourceCardGeneratePageBtn": ".t--generate-template"
+  "generatePageFormSubmitBtn":"[data-cy=t--generate-page-form-submit]"
 }

app/client/cypress/support/commands.js

@@ -2464,6 +2464,9 @@ Cypress.Commands.add("startServerAndRoutes", () => {
   cy.route("GET", "/api/v1/datasources/*/structure?ignoreCache=*").as(
     "getDatasourceStructure",
   );
+  cy.route("PUT", "/api/v1/datasources/datasource-query/*").as(
+    "datasourceQuery",
+  );
 
   cy.route("PUT", "/api/v1/pages/crud-page/*").as("replaceLayoutWithCRUDPage");
   cy.route("POST", "/api/v1/pages/crud-page").as("generateCRUDPage");
@@ -2683,3 +2686,19 @@ Cypress.Commands.add("renameDatasource", (datasourceName) => {
 Cypress.Commands.add("skipGenerateCRUDPage", () => {
   cy.get(generatePage.buildFromScratchActionCard).click();
 });
+
+Cypress.Commands.add("fillAmazonS3DatasourceForm", () => {
+  cy.get(datasourceEditor.projectID).type(Cypress.env("S3_ACCESS_KEY"));
+  cy.get(datasourceEditor.serviceAccCredential)
+    .clear()
+    .type(Cypress.env("S3_SECRET_KEY"));
+});
+
+Cypress.Commands.add("createAmazonS3Datasource", () => {
+  cy.NavigateToDatasourceEditor();
+  cy.get(datasourceEditor.AmazonS3).click();
+
+  cy.fillAmazonS3DatasourceForm();
+
+  cy.testSaveDatasource();
+});

app/client/src/actions/datasourceActions.ts

@@ -199,20 +199,20 @@ export const getOAuthAccessToken = (datasourceId: string) => {
   };
 };
 
-export type executeDatasourceQuerySuccessPayload = {
+export type executeDatasourceQuerySuccessPayload<T> = {
   responseMeta: ResponseMeta;
   data: {
-    body: Array<{ id: string; name: string }>;
+    body: T;
     headers: Record<string, string[]>;
     statusCode: string;
     isExecutionSuccess: boolean;
   };
 };
 type errorPayload = unknown;
 
-export type executeDatasourceQueryReduxAction = ReduxActionWithCallbacks<
+export type executeDatasourceQueryReduxAction<T> = ReduxActionWithCallbacks<
   executeDatasourceQueryRequest,
-  executeDatasourceQuerySuccessPayload,
+  executeDatasourceQuerySuccessPayload<T>,
   errorPayload
 >;
 
@@ -222,9 +222,11 @@ export const executeDatasourceQuery = ({
   payload,
 }: {
   onErrorCallback?: (payload: errorPayload) => void;
-  onSuccessCallback?: (payload: executeDatasourceQuerySuccessPayload) => void;
+  onSuccessCallback?: (
+    payload: executeDatasourceQuerySuccessPayload<any>,
+  ) => void;
   payload: executeDatasourceQueryRequest;
-}): executeDatasourceQueryReduxAction => {
+}): executeDatasourceQueryReduxAction<any> => {
   return {
     type: ReduxActionTypes.EXECUTE_DATASOURCE_QUERY_INIT,
     payload,

app/client/src/api/DatasourcesApi.ts

@@ -25,7 +25,7 @@ export interface EmbeddedRestDatasourceRequest {
   pluginId: string;
 }
 
-type executeQueryData = Array<{ key: string; value?: string }>;
+type executeQueryData = Array<{ key?: string; value?: string }>;
 
 export interface executeDatasourceQueryRequest {
   datasourceId: string;

app/client/src/components/ads/Dropdown.tsx

@@ -66,6 +66,7 @@ export type DropdownProps = CommonComponentProps &
     renderOption?: RenderOption;
     isLoading?: boolean;
     errorMsg?: string; // If errorMsg is defined, we show dropDown's error state with the message.
+    helperText?: string;
   };
 export interface DefaultDropDownValueNodeProps {
   selected: DropdownOption;
@@ -309,14 +310,20 @@ const SelectedIcon = styled(Icon)`
 const ErrorMsg = styled.span`
   ${(props) => getTypographyByKey(props, "p3")};
   color: ${Colors.POMEGRANATE2};
-  margin: 6px 0px 10px;
+  margin-top: 8px;
 `;
 
 const ErrorLabel = styled.span`
   ${(props) => getTypographyByKey(props, "p1")};
   color: ${Colors.POMEGRANATE2};
 `;
 
+const HelperText = styled.span`
+  ${(props) => getTypographyByKey(props, "p3")};
+  color: ${Colors.GRAY};
+  margin-top: 8px;
+`;
+
 function DefaultDropDownValueNode({
   errorMsg,
   renderNode,
@@ -455,12 +462,20 @@ export default function Dropdown(props: DropdownProps) {
     SelectedValueNode = DefaultDropDownValueNode,
     renderOption,
     errorMsg = "",
+    helperText = "",
   } = { ...props };
   const [isOpen, setIsOpen] = useState<boolean>(false);
   const [selected, setSelected] = useState<DropdownOption>(props.selected);
 
+  const closeIfOpen = () => {
+    if (isOpen) {
+      setIsOpen(false);
+    }
+  };
+
   useEffect(() => {
     setSelected(props.selected);
+    closeIfOpen();
   }, [props.selected]);
 
   const optionClickHandler = useCallback(
@@ -474,7 +489,7 @@ export default function Dropdown(props: DropdownProps) {
   );
 
   const disabled = props.disabled || isLoading || !!errorMsg;
-  const downIconColor = errorMsg ? Colors.POMEGRANATE2 : "";
+  const downIconColor = errorMsg ? Colors.POMEGRANATE2 : Colors.DARK_GRAY;
 
   const dropdownTrigger = props.dropdownTriggerIcon ? (
     <DropdownTriggerWrapper
@@ -516,6 +531,9 @@ export default function Dropdown(props: DropdownProps) {
         )}
       </Selected>
       {errorMsg && <ErrorMsg>{errorMsg}</ErrorMsg>}
+      {helperText && !isOpen && !errorMsg && (
+        <HelperText>{helperText}</HelperText>
+      )}
     </DropdownSelect>
   );
   return (