diff --git a/.editorconfig b/.editorconfig index b34968069b9e7..2b66fed8a552f 100644 --- a/.editorconfig +++ b/.editorconfig @@ -402,8 +402,11 @@ ij_xml_space_inside_empty_tag = false ij_xml_text_wrap = normal ij_xml_use_custom_settings = false -[{*.ats,*.ts}] -ij_continuation_indent_size = 4 +[{*.ats,*.ts,*.tsx}] +max_line_length = off +indent_size = 2 +tab_width = 2 +ij_continuation_indent_size = 2 ij_typescript_align_imports = false ij_typescript_align_multiline_array_initializer_expression = false ij_typescript_align_multiline_binary_operation = false diff --git a/dataline-webapp/package-lock.json b/dataline-webapp/package-lock.json index bf172599ae500..d881f8f948342 100644 --- a/dataline-webapp/package-lock.json +++ b/dataline-webapp/package-lock.json @@ -5550,6 +5550,11 @@ "resolved": "https://registry.npmjs.org/date-arithmetic/-/date-arithmetic-3.1.0.tgz", "integrity": "sha1-H80D29UEudvuK5B4yFpfHH08wtM=" }, + "dayjs": { + "version": "1.8.35", + "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.8.35.tgz", + "integrity": "sha512-isAbIEenO4ilm6f8cpqvgjZCsuerDAz2Kb7ri201AiNn58aqXuaLJEnCtfIMdCvERZHNGRY5lDMTr/jdAnKSWQ==" + }, "debug": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", @@ -7906,12 +7911,11 @@ } }, "framesync": { - "version": "4.0.4", - "resolved": "https://registry.npmjs.org/framesync/-/framesync-4.0.4.tgz", - "integrity": "sha512-mdP0WvVHe0/qA62KG2LFUAOiWLng5GLpscRlwzBxu2VXOp6B8hNs5C5XlFigsMgrfDrr2YbqTsgdWZTc4RXRMQ==", + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/framesync/-/framesync-4.1.0.tgz", + "integrity": "sha512-MmgZ4wCoeVxNbx2xp5hN/zPDCbLSKiDt4BbbslK7j/pM2lg5S0vhTNv1v8BCVb99JPIo6hXBFdwzU7Q4qcAaoQ==", "requires": { - "hey-listen": "^1.0.8", - "tslib": "^1.10.0" + "hey-listen": "^1.0.5" } }, "fresh": { @@ -12621,9 +12625,9 @@ } }, "popmotion": { - "version": "8.7.3", - "resolved": "https://registry.npmjs.org/popmotion/-/popmotion-8.7.3.tgz", - "integrity": "sha512-OcpS/V9sCJjrKiVfp3JB5kp5SBqefZ4RvM9GBLYgv0YbULxv9S5METP9ueVJxSClR3yrfFEY2pLWTWKLn/EUfg==", + "version": "8.7.5", + "resolved": "https://registry.npmjs.org/popmotion/-/popmotion-8.7.5.tgz", + "integrity": "sha512-p85l/qrOuLTQZ+aGfyB8cqOzDRWgiSFN941jSrj9CsWeJzUn+jiGSWJ50sr59gWAZ8TKIvqdDowqFlScc0NEyw==", "requires": { "@popmotion/easing": "^1.0.1", "@popmotion/popcorn": "^0.4.4", @@ -12684,14 +12688,9 @@ }, "dependencies": { "@types/node": { - "version": "10.17.28", - "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.28.tgz", - "integrity": "sha512-dzjES1Egb4c1a89C7lKwQh8pwjYmlOAG9dW1pBgxEk57tMrLnssOfEthz8kdkNaBd7lIqQx7APm5+mZ619IiCQ==" - }, - "typescript": { - "version": "3.9.7", - "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==" + "version": "10.17.32", + "resolved": "https://registry.npmjs.org/@types/node/-/node-10.17.32.tgz", + "integrity": "sha512-EUq+cjH/3KCzQHikGnNbWAGe548IFLSm93Vl8xA7EuYEEATiyOVDyEVuGkowL7c9V69FF/RiZSAOCFPApMs/ig==" } } }, @@ -17037,8 +17036,7 @@ "typescript": { "version": "3.9.7", "resolved": "https://registry.npmjs.org/typescript/-/typescript-3.9.7.tgz", - "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==", - "dev": true + "integrity": "sha512-BLbiRkiBzAwsjut4x/dsibSTB6yWpwT5qWmC2OfuCg3GgVQCSgMs4vEctYPhsaGtd0AeuuHMkjZ2h2WG8MSzRw==" }, "uncontrollable": { "version": "5.1.0", diff --git a/dataline-webapp/package.json b/dataline-webapp/package.json index f90a2bf69bef8..28dc3bdfe8af2 100644 --- a/dataline-webapp/package.json +++ b/dataline-webapp/package.json @@ -14,6 +14,7 @@ "@fortawesome/free-regular-svg-icons": "^5.14.0", "@fortawesome/free-solid-svg-icons": "^5.12.1", "@fortawesome/react-fontawesome": "^0.1.8", + "dayjs": "^1.8.35", "formik": "2.1.5", "query-string": "^6.13.1", "react": "^16.12.0", diff --git a/dataline-webapp/src/components/FrequencyForm/FrequencyForm.tsx b/dataline-webapp/src/components/FrequencyForm/FrequencyForm.tsx index 5d966a3cb708f..cbcbcd3cf78b6 100644 --- a/dataline-webapp/src/components/FrequencyForm/FrequencyForm.tsx +++ b/dataline-webapp/src/components/FrequencyForm/FrequencyForm.tsx @@ -1,5 +1,5 @@ -import React from "react"; -import { useIntl } from "react-intl"; +import React, { useState } from "react"; +import { FormattedMessage, useIntl } from "react-intl"; import styled from "styled-components"; import * as yup from "yup"; import { Field, FieldProps, Form, Formik } from "formik"; @@ -7,11 +7,15 @@ import { Field, FieldProps, Form, Formik } from "formik"; import LabeledDropDown from "../LabeledDropDown"; import FrequencyConfig from "../../data/FrequencyConfig.json"; import BottomBlock from "./components/BottomBlock"; +import Label from "../Label"; +import TreeView, { INode } from "../TreeView/TreeView"; type IProps = { className?: string; + schema: INode[]; errorMessage?: React.ReactNode; - onSubmit: (values: { frequency: string }) => void; + onSubmit: (values: { frequency: string }, checkedState: string[]) => void; + initialCheckedSchema: Array; }; const SmallLabeledDropDown = styled(LabeledDropDown)` @@ -22,6 +26,14 @@ const FormContainer = styled(Form)` padding: 22px 27px 23px 24px; `; +const TreeViewContainer = styled.div` + width: 100%; + background: ${({ theme }) => theme.greyColor0}; + margin-bottom: 29px; + border-radius: 4px; + overflow: hidden; +`; + const connectionValidationSchema = yup.object().shape({ frequency: yup.string().required("form.empty.error") }); @@ -29,7 +41,9 @@ const connectionValidationSchema = yup.object().shape({ const FrequencyForm: React.FC = ({ onSubmit, className, - errorMessage + errorMessage, + schema, + initialCheckedSchema }) => { const formatMessage = useIntl().formatMessage; const dropdownData = React.useMemo( @@ -51,6 +65,9 @@ const FrequencyForm: React.FC = ({ [formatMessage] ); + const [checkedState, setCheckedState] = useState(initialCheckedSchema); + const onCheckAction = (data: Array) => setCheckedState(data); + return ( = ({ validateOnChange={true} validationSchema={connectionValidationSchema} onSubmit={async (values, { setSubmitting }) => { - await onSubmit(values); + await onSubmit(values, checkedState); setSubmitting(false); }} > {({ isSubmitting, setFieldValue, isValid, dirty }) => ( - + + + + + {({ field }: FieldProps) => ( ` box-shadow: 0 1px 2px ${({ theme }) => theme.shadowColor}; border-radius: 50%; margin-right: 10px; - padding-top: 1px; + padding-top: 4px; color: ${({ theme }) => theme.whiteColor}; font-size: 12px; + line-height: 12px; text-align: center; display: inline-block; `; diff --git a/dataline-webapp/src/components/TreeView/TreeView.tsx b/dataline-webapp/src/components/TreeView/TreeView.tsx index 184c5fe443575..14606e1b535e9 100644 --- a/dataline-webapp/src/components/TreeView/TreeView.tsx +++ b/dataline-webapp/src/components/TreeView/TreeView.tsx @@ -6,7 +6,7 @@ import { faChevronRight, faCheck } from "@fortawesome/free-solid-svg-icons"; import "react-checkbox-tree/lib/react-checkbox-tree.css"; -type INode = { +export type INode = { value: string; label: string; children?: Array; diff --git a/dataline-webapp/src/config/index.ts b/dataline-webapp/src/config/index.ts index 422a03db7a585..cf9f5c9b9a6b5 100644 --- a/dataline-webapp/src/config/index.ts +++ b/dataline-webapp/src/config/index.ts @@ -1,13 +1,15 @@ const config: { - ui: { helpLink: string; docsLink: string; workspaceId: string }; - apiUrl: string; + ui: { helpLink: string; docsLink: string; workspaceId: string }; + apiUrl: string; } = { - ui: { - helpLink: "https://dataline.io/", - docsLink: "https://docs.dataline.io", - workspaceId: "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6" - }, - apiUrl: process.env.REACT_APP_API_URL || `${window.location.protocol}//${window.location.hostname}:8001/api/v1/` + ui: { + helpLink: "https://dataline.io/", + docsLink: "https://docs.dataline.io", + workspaceId: "5ae6b09b-fdec-41af-aaf7-7d94cfc33ef6" + }, + apiUrl: + process.env.REACT_APP_API_URL || + `${window.location.protocol}//${window.location.hostname}:8001/api/v1/` }; export default config; diff --git a/dataline-webapp/src/core/helpers.tsx b/dataline-webapp/src/core/helpers.tsx new file mode 100644 index 0000000000000..13e79269b05ae --- /dev/null +++ b/dataline-webapp/src/core/helpers.tsx @@ -0,0 +1,43 @@ +import { SyncSchema } from "./resources/Schema"; + +export const constructInitialSchemaState = (syncSchema: SyncSchema) => { + const initialChecked: Array = []; + syncSchema.tables.map(item => + item.columns.forEach(column => + column.selected + ? initialChecked.push(`${item.name}_${column.name}`) + : null + ) + ); + + const formSyncSchema = syncSchema.tables.map((item: any) => ({ + value: item.name, + label: item.name, + children: item.columns.map((column: any) => ({ + value: `${item.name}_${column.name}`, + label: column.name + })) + })); + + return { + formSyncSchema, + initialChecked + }; +}; + +export const constructNewSchema = ( + syncSchema: SyncSchema, + checkedState: string[] +) => { + const newSyncSchema = { + tables: syncSchema.tables.map(item => ({ + ...item, + columns: item.columns.map(column => ({ + ...column, + selected: checkedState.includes(`${item.name}_${column.name}`) + })) + })) + }; + + return newSyncSchema; +}; diff --git a/dataline-webapp/src/core/resources/Connection.ts b/dataline-webapp/src/core/resources/Connection.ts index 67eaeb450278b..bc189f2f2f308 100644 --- a/dataline-webapp/src/core/resources/Connection.ts +++ b/dataline-webapp/src/core/resources/Connection.ts @@ -8,7 +8,7 @@ export type ScheduleProperties = { export type SyncSchemaColumn = { name: string; - selected: string; + selected: boolean; type: string; }; @@ -142,4 +142,20 @@ export default class ConnectionResource extends BaseResource } }; } + + static syncShape(this: T) { + return { + ...super.detailShape(), + getFetchKey: (params: any) => + "POST " + this.url(params) + "/sync" + JSON.stringify(params), + fetch: async ( + params: Readonly> + ): Promise => { + await this.fetch("post", `${this.url(params)}/sync`, params); + return { + connectionId: params.connectionId + }; + } + }; + } } diff --git a/dataline-webapp/src/core/resources/Job.ts b/dataline-webapp/src/core/resources/Job.ts new file mode 100644 index 0000000000000..daaea44ae93f7 --- /dev/null +++ b/dataline-webapp/src/core/resources/Job.ts @@ -0,0 +1,52 @@ +import { Resource, FetchOptions } from "rest-hooks"; +import BaseResource from "./BaseResource"; +import JobLogsResource from "./JobLogs"; + +export interface Job { + id: number; + configType: string; + configId: string; + createdAt: number; + startedAt: number; + updatedAt: number; + status: string; +} + +export default class JobResource extends BaseResource implements Job { + readonly id: number = 0; + readonly configType: string = ""; + readonly configId: string = ""; + readonly createdAt: number = 0; + readonly startedAt: number = 0; + readonly updatedAt: number = 0; + readonly status: string = ""; + + pk() { + return this.id?.toString(); + } + + static urlRoot = "jobs"; + + static getFetchOptions(): FetchOptions { + return { + pollFrequency: 2500 // every 2,5 seconds + }; + } + + static listShape(this: T) { + return { + ...super.listShape(), + schema: { jobs: [this.asSchema()] } + }; + } + + static detailShape(this: T) { + return { + ...super.detailShape(), + schema: { + job: this.asSchema(), + logs: JobLogsResource.asSchema() + } + }; + } +} diff --git a/dataline-webapp/src/core/resources/JobLogs.ts b/dataline-webapp/src/core/resources/JobLogs.ts new file mode 100644 index 0000000000000..debf8e696994a --- /dev/null +++ b/dataline-webapp/src/core/resources/JobLogs.ts @@ -0,0 +1,17 @@ +import BaseResource from "./BaseResource"; + +export interface JobLogs { + stdout: string[]; + stderr: string[]; +} + +export default class JobLogsResource extends BaseResource implements JobLogs { + readonly stdout: string[] = []; + readonly stderr: string[] = []; + + pk() { + return ""; + } + + static urlRoot = "jobs"; +} diff --git a/dataline-webapp/src/core/resources/Schema.ts b/dataline-webapp/src/core/resources/Schema.ts new file mode 100644 index 0000000000000..4a48786514c6f --- /dev/null +++ b/dataline-webapp/src/core/resources/Schema.ts @@ -0,0 +1,53 @@ +import { Resource } from "rest-hooks"; +import BaseResource from "./BaseResource"; + +export type SyncSchemaColumn = { + name: string; + selected: boolean; + type: string; +}; + +export type SyncSchema = { + tables: { + name: string; + columns: SyncSchemaColumn[]; + }[]; +}; + +export interface Schema { + id: string; + schema: SyncSchema; +} + +export default class SchemaResource extends BaseResource implements Schema { + readonly schema: SyncSchema = { tables: [] }; + readonly id: string = ""; + + pk() { + return this.id?.toString(); + } + + static urlRoot = "source_implementations"; + + static schemaShape(this: T) { + return { + ...super.detailShape(), + getFetchKey: (params: { sourceImplementationId: string }) => + `POST /source_implementations/discover_schema` + JSON.stringify(params), + fetch: async ( + params: Readonly> + ): Promise => { + const result = await this.fetch( + "post", + `${this.url(params)}/discover_schema`, + params + ); + return { + schema: result?.schema, + id: params.sourceImplementationId + }; + }, + schema: this.asSchema() + }; + } +} diff --git a/dataline-webapp/src/data/FrequencyConfig.json b/dataline-webapp/src/data/FrequencyConfig.json index c0619dbd841ca..51f8b6dbeaf12 100644 --- a/dataline-webapp/src/data/FrequencyConfig.json +++ b/dataline-webapp/src/data/FrequencyConfig.json @@ -2,10 +2,7 @@ { "text": "manual", "value": "manual", - "config": { - "units": 0, - "timeUnit": "minutes" - } + "config": null }, { "text": "5 min", diff --git a/dataline-webapp/src/locales/en.json b/dataline-webapp/src/locales/en.json index 74f0f8b22d8c5..14e86ad28d6eb 100644 --- a/dataline-webapp/src/locales/en.json +++ b/dataline-webapp/src/locales/en.json @@ -25,6 +25,8 @@ "form.frequency": "Sync frequency", "form.frequency.placeholder": "Select a frequency", "form.frequency.message": "Set how often Dataline attempts to replicate data.", + "form.dataSync": "Select the data you want to sync", + "form.dataSync.message": "You’ll be able to change this later on.", "form.cancel": "Cancel", "form.delete": "Delete", "form.saveChanges": "Save changes", @@ -99,6 +101,15 @@ "sources.dataDelete": "No data will be deleted from your source.", "sources.deleteSource": "Delete this source", "sources.deleteConfirm": "Confirm source deletion", + "sources.failed": "Failed", + "sources.completed": "Completed", + "sources.pending": "Pending", + "sources.running": "Running", + "sources.cancelled": "Cancelled", + "sources.emptyLogs": "Empty data", + "sources.hour": "{hour}h ", + "sources.minute": "{minute}m ", + "sources.second": "{second}s", "sources.deleteModalText": "This can not be un-done without a full re-sync. Note that:\n - All past logs and configurations will be deleted\n - Updates of new data will stop\n - No existing data in the destination will be altered", "destination.destinationSettings": "Destination Settings" diff --git a/dataline-webapp/src/pages/OnboardingPage/OnboardingPage.tsx b/dataline-webapp/src/pages/OnboardingPage/OnboardingPage.tsx index cd34c174a338b..18b6f5cac558e 100644 --- a/dataline-webapp/src/pages/OnboardingPage/OnboardingPage.tsx +++ b/dataline-webapp/src/pages/OnboardingPage/OnboardingPage.tsx @@ -18,6 +18,7 @@ import FrequencyConfig from "../../data/FrequencyConfig.json"; import { Routes } from "../routes"; import useRouter from "../../components/hooks/useRouterHook"; import { Source } from "../../core/resources/Source"; +import { SyncSchema } from "../../core/resources/Schema"; const Content = styled.div` width: 100%; @@ -171,6 +172,7 @@ const OnboardingPage: React.FC = () => { const onSubmitConnectionStep = async (values: { frequency: string; + syncSchema: SyncSchema; source?: Source; }) => { const frequencyData = FrequencyConfig.find( @@ -190,7 +192,8 @@ const OnboardingPage: React.FC = () => { destinations[0].destinationImplementationId, syncMode: "full_refresh", schedule: frequencyData?.config, - status: "active" + status: "active", + syncSchema: values.syncSchema }, [ [ @@ -240,6 +243,7 @@ const OnboardingPage: React.FC = () => { currentSourceId={sources[0].sourceId} currentDestinationId={destinations[0].destinationId} errorStatus={errorStatusRequest} + sourceImplementationId={sources[0].sourceImplementationId} /> ); }; diff --git a/dataline-webapp/src/pages/OnboardingPage/components/ConnectionForm.tsx b/dataline-webapp/src/pages/OnboardingPage/components/ConnectionForm.tsx new file mode 100644 index 0000000000000..a3a55a06946df --- /dev/null +++ b/dataline-webapp/src/pages/OnboardingPage/components/ConnectionForm.tsx @@ -0,0 +1,48 @@ +import React from "react"; +import { useResource } from "rest-hooks"; + +import FrequencyForm from "../../../components/FrequencyForm"; +import SchemaResource, { SyncSchema } from "../../../core/resources/Schema"; +import { + constructInitialSchemaState, + constructNewSchema +} from "../../../core/helpers"; + +type IProps = { + onSubmit: (values: { frequency: string; syncSchema: SyncSchema }) => void; + errorMessage?: React.ReactNode; + sourceImplementationId: string; +}; + +const ConnectionStep: React.FC = ({ + onSubmit, + errorMessage, + sourceImplementationId +}) => { + const { schema } = useResource(SchemaResource.schemaShape(), { + sourceImplementationId + }); + + const { formSyncSchema, initialChecked } = constructInitialSchemaState( + schema + ); + + const onSubmitForm = async ( + values: { frequency: string }, + checkedState: string[] + ) => { + const newSchema = constructNewSchema(schema, checkedState); + await onSubmit({ ...values, syncSchema: newSchema }); + }; + + return ( + + ); +}; + +export default ConnectionStep; diff --git a/dataline-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx b/dataline-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx index 6f4db442a27e6..7f6d82863ea9d 100644 --- a/dataline-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx +++ b/dataline-webapp/src/pages/OnboardingPage/components/ConnectionStep.tsx @@ -1,25 +1,39 @@ -import React from "react"; +import React, { Suspense } from "react"; import { FormattedMessage } from "react-intl"; import { useResource } from "rest-hooks"; +import styled from "styled-components"; import ContentCard from "../../../components/ContentCard"; import ConnectionBlock from "../../../components/ConnectionBlock"; -import FrequencyForm from "../../../components/FrequencyForm"; +import ConnectionForm from "./ConnectionForm"; import SourceResource, { Source } from "../../../core/resources/Source"; import DestinationResource from "../../../core/resources/Destination"; +import Spinner from "../../../components/Spinner"; +import { SyncSchema } from "../../../core/resources/Schema"; type IProps = { - onSubmit: (values: { frequency: string; source: Source }) => void; + onSubmit: (values: { + frequency: string; + syncSchema: SyncSchema; + source: Source; + }) => void; currentSourceId: string; currentDestinationId: string; + sourceImplementationId: string; errorStatus?: number; }; +const SpinnerBlock = styled.div` + margin: 40px; + text-align: center; +`; + const ConnectionStep: React.FC = ({ onSubmit, currentSourceId, currentDestinationId, - errorStatus + errorStatus, + sourceImplementationId }) => { const currentSource = useResource(SourceResource.detailShape(), { sourceId: currentSourceId @@ -28,7 +42,10 @@ const ConnectionStep: React.FC = ({ destinationId: currentDestinationId }); - const onSubmitStep = async (values: { frequency: string }) => { + const onSubmitStep = async (values: { + frequency: string; + syncSchema: SyncSchema; + }) => { await onSubmit({ ...values, source: { @@ -51,7 +68,19 @@ const ConnectionStep: React.FC = ({ itemTo={{ name: currentDestination.name }} /> }> - + + + + } + > + + ); diff --git a/dataline-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/FrequencyCell.tsx b/dataline-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/FrequencyCell.tsx index 35f964730d4e6..d3803c6190f5a 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/FrequencyCell.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/AllSourcesPage/components/FrequencyCell.tsx @@ -14,9 +14,7 @@ const Content = styled.div<{ enabled?: boolean }>` const FrequencyCell: React.FC = ({ value, enabled }) => { const cellText = FrequencyConfig.find( - item => - item.config.units === value?.units && - item.config.timeUnit === value?.timeUnit + item => JSON.stringify(item.config) === JSON.stringify(value) ); return {cellText?.text || ""}; }; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/CreateSourcePage.tsx b/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/CreateSourcePage.tsx index abf48fbc96df3..6cf3165314166 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/CreateSourcePage.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/CreateSourcePage.tsx @@ -18,6 +18,7 @@ import SourceImplementationResource, { } from "../../../../core/resources/SourceImplementation"; import FrequencyConfig from "../../../../data/FrequencyConfig.json"; import ConnectionResource from "../../../../core/resources/Connection"; +import { SyncSchema } from "../../../../core/resources/Schema"; const Content = styled.div` max-width: 638px; @@ -104,7 +105,10 @@ const CreateSourcePage: React.FC = () => { setErrorStatusRequest(e.status); } }; - const onSubmitConnectionStep = async (values: { frequency: string }) => { + const onSubmitConnectionStep = async (values: { + frequency: string; + syncSchema: SyncSchema; + }) => { const frequencyData = FrequencyConfig.find( item => item.value === values.frequency ); @@ -126,7 +130,8 @@ const CreateSourcePage: React.FC = () => { currentDestination.destinationImplementationId, syncMode: "full_refresh", schedule: frequencyData?.config, - status: "active" + status: "active", + syncSchema: values.syncSchema }, [ [ @@ -164,6 +169,9 @@ const CreateSourcePage: React.FC = () => { onSubmit={onSubmitConnectionStep} destination={destination} sourceId={currentSourceImplementation?.sourceId || ""} + sourceImplementationId={ + currentSourceImplementation?.sourceImplementationId || "" + } /> ); }; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionForm.tsx b/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionForm.tsx new file mode 100644 index 0000000000000..252f839e9d8a9 --- /dev/null +++ b/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionForm.tsx @@ -0,0 +1,47 @@ +import React from "react"; +import { useResource } from "rest-hooks"; + +import FrequencyForm from "../../../../../components/FrequencyForm"; +import SchemaResource, { + SyncSchema +} from "../../../../../core/resources/Schema"; +import { + constructInitialSchemaState, + constructNewSchema +} from "../../../../../core/helpers"; + +type IProps = { + onSubmit: (values: { frequency: string; syncSchema: SyncSchema }) => void; + sourceImplementationId: string; +}; + +const ConnectionStep: React.FC = ({ + onSubmit, + sourceImplementationId +}) => { + const { schema } = useResource(SchemaResource.schemaShape(), { + sourceImplementationId + }); + + const { formSyncSchema, initialChecked } = constructInitialSchemaState( + schema + ); + + const onSubmitForm = async ( + values: { frequency: string }, + checkedState: string[] + ) => { + const newSchema = constructNewSchema(schema, checkedState); + await onSubmit({ ...values, syncSchema: newSchema }); + }; + + return ( + + ); +}; + +export default ConnectionStep; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionStep.tsx b/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionStep.tsx index 72b470052228c..a8dfaa1ce3dde 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionStep.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/CreateSourcePage/components/ConnectionStep.tsx @@ -1,27 +1,38 @@ -import React from "react"; +import React, { Suspense } from "react"; import { FormattedMessage } from "react-intl"; import { useResource } from "rest-hooks"; +import styled from "styled-components"; import ConnectionBlock from "../../../../../components/ConnectionBlock"; import ContentCard from "../../../../../components/ContentCard"; -import FrequencyForm from "../../../../../components/FrequencyForm"; import { Destination } from "../../../../../core/resources/Destination"; import SourceResource from "../../../../../core/resources/Source"; +import Spinner from "../../../../../components/Spinner"; +import { SyncSchema } from "../../../../../core/resources/Schema"; +import ConnectionForm from "./ConnectionForm"; type IProps = { - onSubmit: (values: { frequency: string }) => void; + onSubmit: (values: { frequency: string; syncSchema: SyncSchema }) => void; destination: Destination; sourceId: string; + sourceImplementationId: string; }; +const SpinnerBlock = styled.div` + margin: 40px; + text-align: center; +`; + const CreateSourcePage: React.FC = ({ onSubmit, destination, - sourceId + sourceId, + sourceImplementationId }) => { const source = useResource(SourceResource.detailShape(), { sourceId }); + return ( <> = ({ itemTo={{ name: destination.name }} /> }> - + + + + } + > + + ); diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx index 375f65e84c8b8..48afa912af797 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/SourceItemPage.tsx @@ -1,6 +1,7 @@ -import React, { useState } from "react"; +import React, { Suspense, useState } from "react"; import { FormattedMessage } from "react-intl"; import { useFetcher, useResource } from "rest-hooks"; +import styled from "styled-components"; import PageTitle from "../../../../components/PageTitle"; import Breadcrumbs from "../../../../components/Breadcrumbs"; @@ -11,6 +12,19 @@ import StatusView from "./components/StatusView"; import SettingsView from "./components/SettingsView"; import SchemaView from "./components/SchemaView"; import ConnectionResource from "../../../../core/resources/Connection"; +import LoadingPage from "../../../../components/LoadingPage"; + +const Content = styled.div` + overflow-y: auto; + height: calc(100% - 67px); + margin-top: -17px; + padding-top: 17px; +`; + +const Page = styled.div` + overflow-y: hidden; + height: 100%; +`; const SourceItemPage: React.FC = () => { const { query, push, history } = useRouter(); @@ -47,7 +61,7 @@ const SourceItemPage: React.FC = () => { name: , onClick: onClickBack }, - { name: connection.name } + { name: connection.source?.name } ]; const onChangeStatus = async () => { @@ -82,7 +96,7 @@ const SourceItemPage: React.FC = () => { }; return ( - <> + } @@ -95,8 +109,10 @@ const SourceItemPage: React.FC = () => { /> } /> - {renderStep()} - + + }>{renderStep()} + + ); }; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobItem.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobItem.tsx new file mode 100644 index 0000000000000..ea73d867487f8 --- /dev/null +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobItem.tsx @@ -0,0 +1,187 @@ +import React, { Suspense, useState } from "react"; +import pose from "react-pose"; +import { + FormattedMessage, + FormattedDateParts, + FormattedTimeParts +} from "react-intl"; +import styled from "styled-components"; +import dayjs from "dayjs"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { faAngleDown } from "@fortawesome/free-solid-svg-icons"; + +import { Job } from "../../../../../core/resources/Job"; +import { Row, Cell } from "../../../../../components/SimpleTableComponents"; +import StatusIcon from "../../../../../components/StatusIcon"; +import Spinner from "../../../../../components/Spinner"; +import JobLogs from "./JobLogs"; + +type IProps = { + job: Job; +}; + +const Item = styled.div<{ isFailed: boolean }>` + border-bottom: 1px solid ${({ theme }) => theme.greyColor20}; + font-size: 15px; + line-height: 18px; + + &:hover { + background: ${({ theme, isFailed }) => + isFailed ? theme.dangerTransparentColor : theme.greyColor0}; + } +`; + +const MainInfo = styled(Row)<{ + isOpen?: boolean; + isFailed?: boolean; +}>` + cursor: pointer; + height: 59px; + padding: 10px 44px 10px 40px; + border-bottom: 1px solid + ${({ theme, isOpen, isFailed }) => + !isOpen + ? "none" + : isFailed + ? theme.dangerTransparentColor + : theme.greyColor20}; +`; + +const Title = styled.div<{ isFailed: boolean }>` + position: relative; + color: ${({ theme, isFailed }) => + isFailed ? theme.dangerColor : theme.darkPrimaryColor}; +`; + +const ErrorSign = styled(StatusIcon)` + position: absolute; + left: -30px; +`; + +const LoadLogs = styled.div` + background: ${({ theme }) => theme.whiteColor}; + text-align: center; + padding: 6px 0; + min-height: 58px; +`; + +const CompletedTime = styled.div` + font-size: 12px; + line-height: 15px; + color: ${({ theme }) => theme.greyColor40}; +`; + +const Arrow = styled.div<{ + isOpen?: boolean; + isFailed?: boolean; +}>` + transform: ${({ isOpen }) => !isOpen && "rotate(-90deg)"}; + transition: 0.3s; + font-size: 22px; + line-height: 22px; + height: 22px; + color: ${({ theme, isFailed }) => + isFailed ? theme.dangerColor : theme.darkPrimaryColor}; + position: absolute; + right: 18px; + top: calc(50% - 11px); + opacity: 0; + + div:hover > div > &, + div:hover > div > div > &, + div:hover > & { + opacity: 1; + } +`; + +const itemConfig = { + open: { + height: "auto", + opacity: 1, + transition: "tween" + }, + closed: { + height: "1px", + opacity: 0, + transition: "tween" + } +}; + +const ContentWrapper = pose.div(itemConfig); + +const JobItem: React.FC = ({ job }) => { + const [isOpen, setIsOpen] = useState(false); + const onExpand = () => setIsOpen(!isOpen); + + const date1 = dayjs(job.createdAt * 1000); + const date2 = dayjs(job.updatedAt * 1000); + const hours = Math.abs(date2.diff(date1, "hour")); + const minutes = Math.abs(date2.diff(date1, "minute")) - hours * 60; + const seconds = + Math.abs(date2.diff(date1, "second")) - minutes * 60 - hours * 3600; + + const isFailed = job.status === "failed"; + return ( + + + + + {isFailed && <ErrorSign />} + <FormattedMessage id={`sources.${job.status}`} /> + + + + + {parts => ( + {`${parts[0].value}:${parts[2].value}${parts[4].value} `} + )} + + + {parts => {`${parts[0].value}/${parts[2].value}`}} + + + {hours ? ( + + ) : null} + {hours || minutes ? ( + + ) : null} + + + + + + + + +
+ + + + } + > + {isOpen && } + +
+ + + ); +}; + +export default JobItem; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobLogs.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobLogs.tsx new file mode 100644 index 0000000000000..f061e60a35d89 --- /dev/null +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobLogs.tsx @@ -0,0 +1,40 @@ +import React from "react"; +import { useResource } from "rest-hooks"; +import { FormattedMessage } from "react-intl"; +import styled from "styled-components"; + +import JobResource from "../../../../../core/resources/Job"; + +type IProps = { + id: number; +}; + +const Logs = styled.div` + padding: 20px 42px; + font-size: 15px; + line-height: 18px; + color: ${({ theme }) => theme.darkPrimaryColor}; +`; + +const JobLogs: React.FC = ({ id }) => { + const job = useResource(JobResource.detailShape(), { id }); + + if (!job.logs.stderr.length) { + return ( + + + + ); + } + + // now logs always empty. TODO: Test ui with data + return ( + + {job.logs.stderr.map((item, key) => ( +
{item}
+ ))} + + ); +}; + +export default JobLogs; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobsList.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobsList.tsx new file mode 100644 index 0000000000000..53c3f17503ea5 --- /dev/null +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/JobsList.tsx @@ -0,0 +1,23 @@ +import React from "react"; +import styled from "styled-components"; + +import JobItem from "./JobItem"; +import { Job } from "../../../../../core/resources/Job"; + +type IProps = { + jobs: Job[]; +}; + +const Content = styled.div``; + +const JobsList: React.FC = ({ jobs }) => { + return ( + + {jobs.map(item => ( + + ))} + + ); +}; + +export default JobsList; diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SchemaView.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SchemaView.tsx index cb32c14cb2043..285d4aea2ce7c 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SchemaView.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SchemaView.tsx @@ -10,6 +10,10 @@ import ConnectionResource, { SyncSchema } from "../../../../../core/resources/Connection"; import EmptySyncHistory from "./EmptySyncHistory"; +import { + constructInitialSchemaState, + constructNewSchema +} from "../../../../../core/helpers"; type IProps = { connectionId: string; @@ -37,29 +41,14 @@ const SchemaView: React.FC = ({ connectionStatus }) => { const updateConnection = useFetcher(ConnectionResource.updateShape()); - const initialChecked: Array = []; - syncSchema.tables.map(item => - item.columns.forEach(column => - column.selected ? initialChecked.push(column.name) : null - ) + const { formSyncSchema, initialChecked } = useMemo( + () => constructInitialSchemaState(syncSchema), + [syncSchema] ); const [disabledButtons, setDisabledButtons] = useState(true); const [checkedState, setCheckedState] = useState(initialChecked); - const formSyncSchema = useMemo( - () => - syncSchema.tables.map((item: any) => ({ - value: item.name, - label: item.name, - children: item.columns.map((column: any) => ({ - value: column.name, - label: column.name - })) - })), - [syncSchema.tables] - ); - const onCheckAction = (data: Array) => { setDisabledButtons(JSON.stringify(data) === JSON.stringify(initialChecked)); setCheckedState(data); @@ -71,15 +60,7 @@ const SchemaView: React.FC = ({ }; const onSubmit = async () => { setDisabledButtons(true); - const newSyncSchema = { - tables: syncSchema.tables.map(item => ({ - ...item, - columns: item.columns.map(column => ({ - ...column, - selected: checkedState.includes(column.name) - })) - })) - }; + const newSyncSchema = constructNewSchema(syncSchema, checkedState); await updateConnection( {}, diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SettingsView.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SettingsView.tsx index 0728d8ec16bf7..d23fbe3318f2c 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SettingsView.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/SettingsView.tsx @@ -41,9 +41,7 @@ const SettingsView: React.FC = ({ sourceData }) => { ); const schedule = FrequencyConfig.find( - item => - item.config.units === sourceData.schedule?.units && - item.config.timeUnit === sourceData.schedule?.timeUnit + item => JSON.stringify(item.config) === JSON.stringify(sourceData.schedule) ); const onSubmit = async (values: { @@ -81,6 +79,7 @@ const SettingsView: React.FC = ({ sourceData }) => { {}, { ...sourceData, + schedule: frequencyData?.config, source: { ...sourceData.source, name: values.name, diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusMainInfo.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusMainInfo.tsx index 0d9c5ed7ab0e7..95749dc72f362 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusMainInfo.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusMainInfo.tsx @@ -67,9 +67,7 @@ const StatusMainInfo: React.FC = ({ sourceData, onEnabledChange }) => { }); const cellText = FrequencyConfig.find( - item => - item.config.units === sourceData.schedule?.units && - item.config.timeUnit === sourceData.schedule?.timeUnit + item => JSON.stringify(item.config) === JSON.stringify(sourceData.schedule) ); return ( diff --git a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusView.tsx b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusView.tsx index 2cafa499660c1..cea90168deff3 100644 --- a/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusView.tsx +++ b/dataline-webapp/src/pages/SourcesPage/pages/SourceItemPage/components/StatusView.tsx @@ -3,12 +3,17 @@ import { FormattedMessage } from "react-intl"; import styled from "styled-components"; import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; import { faRedoAlt } from "@fortawesome/free-solid-svg-icons"; +import { useFetcher, useSubscription, useResource } from "rest-hooks"; import ContentCard from "../../../../../components/ContentCard"; import Button from "../../../../../components/Button"; import StatusMainInfo from "./StatusMainInfo"; import EmptySyncHistory from "./EmptySyncHistory"; -import { Connection } from "../../../../../core/resources/Connection"; +import ConnectionResource, { + Connection +} from "../../../../../core/resources/Connection"; +import JobResource from "../../../../../core/resources/Job"; +import JobsList from "./JobsList"; type IProps = { sourceData: Connection; @@ -20,6 +25,10 @@ const Content = styled.div` margin: 18px auto; `; +const StyledContentCard = styled(ContentCard)` + margin-bottom: 20px; +`; + const Title = styled.div` display: flex; justify-content: space-between; @@ -38,25 +47,41 @@ const SyncButton = styled(Button)` `; const StatusView: React.FC = ({ sourceData, onEnabledChange }) => { + const { jobs } = useResource(JobResource.listShape(), { + configId: sourceData.connectionId, + configType: "sync" + }); + useSubscription(JobResource.listShape(), { + configId: sourceData.connectionId, + configType: "sync" + }); + + const SyncConnection = useFetcher(ConnectionResource.syncShape()); + + const onSync = () => + SyncConnection({ + connectionId: sourceData.connectionId + }); + return ( - - + } > - - + {jobs.length ? : } + ); };