diff --git a/web/web/src/app/metalakes/metalake/rightContent/CreateFilesetDialog.js b/web/web/src/app/metalakes/metalake/rightContent/CreateFilesetDialog.js new file mode 100644 index 00000000000..cb19015458b --- /dev/null +++ b/web/web/src/app/metalakes/metalake/rightContent/CreateFilesetDialog.js @@ -0,0 +1,510 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +'use client' + +import { useState, forwardRef, useEffect, Fragment } from 'react' + +import { + Box, + Grid, + Button, + Dialog, + TextField, + Typography, + DialogContent, + DialogActions, + IconButton, + Fade, + Select, + MenuItem, + InputLabel, + FormControl, + FormHelperText +} from '@mui/material' + +import Icon from '@/components/Icon' + +import { useAppDispatch } from '@/lib/hooks/useStore' +import { createFileset, updateFileset } from '@/lib/store/metalakes' + +import * as yup from 'yup' +import { useForm, Controller } from 'react-hook-form' +import { yupResolver } from '@hookform/resolvers/yup' + +import { groupBy } from 'lodash-es' +import { genUpdates } from '@/lib/utils' +import { nameRegex, nameRegexDesc, keyRegex } from '@/lib/utils/regex' +import { useSearchParams } from 'next/navigation' + +const defaultValues = { + name: '', + type: 'managed', + storageLocation: '', + comment: '', + propItems: [] +} + +const schema = yup.object().shape({ + name: yup.string().required().matches(nameRegex, nameRegexDesc), + type: yup.mixed().oneOf(['managed', 'external']).required(), + storageLocation: yup.string().when('type', { + is: 'external', + then: schema => schema.required(), + otherwise: schema => schema + }), + propItems: yup.array().of( + yup.object().shape({ + required: yup.boolean(), + key: yup.string().required(), + value: yup.string().when('required', { + is: true, + then: schema => schema.required() + }) + }) + ) +}) + +const Transition = forwardRef(function Transition(props, ref) { + return +}) + +const CreateFilesetDialog = props => { + const { open, setOpen, type = 'create', data = {} } = props + const searchParams = useSearchParams() + const metalake = searchParams.get('metalake') + const catalog = searchParams.get('catalog') + const catalogType = searchParams.get('type') + const schemaName = searchParams.get('schema') + const [innerProps, setInnerProps] = useState([]) + const dispatch = useAppDispatch() + const [cacheData, setCacheData] = useState() + + const { + control, + reset, + watch, + setValue, + getValues, + handleSubmit, + trigger, + formState: { errors } + } = useForm({ + defaultValues, + mode: 'all', + resolver: yupResolver(schema) + }) + + const handleFormChange = ({ index, event }) => { + let data = [...innerProps] + data[index][event.target.name] = event.target.value + + if (event.target.name === 'key') { + const invalidKey = !keyRegex.test(event.target.value) + data[index].invalid = invalidKey + } + + const nonEmptyKeys = data.filter(item => item.key.trim() !== '') + const grouped = groupBy(nonEmptyKeys, 'key') + const duplicateKeys = Object.keys(grouped).some(key => grouped[key].length > 1) + + if (duplicateKeys) { + data[index].hasDuplicateKey = duplicateKeys + } else { + data.forEach(it => (it.hasDuplicateKey = false)) + } + + setInnerProps(data) + setValue('propItems', data) + } + + const addFields = () => { + const duplicateKeys = innerProps + .filter(item => item.key.trim() !== '') + .some( + (item, index, filteredItems) => + filteredItems.findIndex(otherItem => otherItem !== item && otherItem.key.trim() === item.key.trim()) !== -1 + ) + + if (duplicateKeys) { + return + } + + let newField = { key: '', value: '', required: false } + + setInnerProps([...innerProps, newField]) + setValue('propItems', [...innerProps, newField]) + } + + const removeFields = index => { + let data = [...innerProps] + data.splice(index, 1) + setInnerProps(data) + setValue('propItems', data) + } + + const handleClose = () => { + reset() + setInnerProps([]) + setValue('propItems', []) + setOpen(false) + } + + const handleClickSubmit = e => { + e.preventDefault() + + return handleSubmit(onSubmit(getValues()), onError) + } + + const onSubmit = data => { + const duplicateKeys = innerProps + .filter(item => item.key.trim() !== '') + .some( + (item, index, filteredItems) => + filteredItems.findIndex(otherItem => otherItem !== item && otherItem.key.trim() === item.key.trim()) !== -1 + ) + + const invalidKeys = innerProps.some(i => i.invalid) + + if (duplicateKeys || invalidKeys) { + return + } + + trigger() + + schema + .validate(data) + .then(() => { + const properties = innerProps.reduce((acc, item) => { + acc[item.key] = item.value + + return acc + }, {}) + + const filesetData = { + name: data.name, + type: data.type, + storageLocation: data.storageLocation, + comment: data.comment, + properties + } + + if (type === 'create') { + dispatch(createFileset({ data: filesetData, metalake, catalog, type: catalogType, schema: schemaName })).then( + res => { + if (!res.payload?.err) { + handleClose() + } + } + ) + } else { + const reqData = { updates: genUpdates(cacheData, filesetData) } + + if (reqData.updates.length !== 0) { + dispatch( + updateFileset({ + metalake, + catalog, + type: catalogType, + schema: schemaName, + fileset: cacheData.name, + data: reqData + }) + ).then(res => { + if (!res.payload?.err) { + handleClose() + } + }) + } + } + }) + .catch(err => { + console.error('valid error', err) + }) + } + + const onError = errors => { + console.error('fields error', errors) + } + + useEffect(() => { + if (open && JSON.stringify(data) !== '{}') { + const { properties = {} } = data + + setCacheData(data) + setValue('name', data.name) + setValue('type', data.type) + setValue('storageLocation', data.storageLocation) + setValue('comment', data.comment) + + const propsItems = Object.entries(properties).map(([key, value]) => { + return { + key, + value + } + }) + + setInnerProps(propsItems) + setValue('propItems', propsItems) + } + }, [open, data, setValue, type]) + + return ( + +
handleClickSubmit(e)}> + `${theme.spacing(8)} !important`, + px: theme => [`${theme.spacing(5)} !important`, `${theme.spacing(15)} !important`], + pt: theme => [`${theme.spacing(8)} !important`, `${theme.spacing(12.5)} !important`] + }} + > + handleClose()} + sx={{ position: 'absolute', right: '1rem', top: '1rem' }} + > + + + + + {type === 'create' ? 'Create' : 'Edit'} Fileset + + + + + + + ( + + )} + /> + {errors.name && {errors.name.message}} + + + + + + + Type + + ( + + )} + /> + {errors.type && {errors.type.message}} + + + + + + ( + + )} + /> + {errors.storageLocation ? ( + {errors.storageLocation.message} + ) : ( + <> + + It is optional if the fileset is 'Managed' type and a storage location is already specified at the + parent catalog or schema level. + + + It becomes mandatory if the fileset type is 'External' or no storage location is defined at the + parent level. + + + )} + + + + + + ( + + )} + /> + + + + + + Properties + + {innerProps.map((item, index) => { + return ( + + + + + + + handleFormChange({ index, event })} + error={item.hasDuplicateKey || item.invalid || !item.key.trim()} + data-refer={`props-key-${index}`} + /> + + + handleFormChange({ index, event })} + data-refer={`props-value-${index}`} + data-prev-refer={`props-${item.key}`} + /> + + + {!(item.disabled || (item.key === 'location' && type === 'update')) ? ( + + removeFields(index)}> + + + + ) : ( + + )} + + + + {item.description} + + {item.hasDuplicateKey && ( + Key already exists + )} + {item.key && item.invalid && ( + + Invalid key, matches strings starting with a letter/underscore, followed by alphanumeric + characters, underscores, hyphens, or dots. + + )} + {!item.key.trim() && ( + Key is required field + )} + + + + ) + })} + + + + + + + + [`${theme.spacing(5)} !important`, `${theme.spacing(15)} !important`], + pb: theme => [`${theme.spacing(5)} !important`, `${theme.spacing(12.5)} !important`] + }} + > + + + +
+
+ ) +} + +export default CreateFilesetDialog diff --git a/web/web/src/app/metalakes/metalake/rightContent/RightContent.js b/web/web/src/app/metalakes/metalake/rightContent/RightContent.js index 1706399ddc2..4dfd091a4b5 100644 --- a/web/web/src/app/metalakes/metalake/rightContent/RightContent.js +++ b/web/web/src/app/metalakes/metalake/rightContent/RightContent.js @@ -26,6 +26,7 @@ import Icon from '@/components/Icon' import MetalakePath from './MetalakePath' import CreateCatalogDialog from './CreateCatalogDialog' import CreateSchemaDialog from './CreateSchemaDialog' +import CreateFilesetDialog from './CreateFilesetDialog' import TabsContent from './tabsContent/TabsContent' import { useSearchParams } from 'next/navigation' import { useAppSelector } from '@/lib/hooks/useStore' @@ -33,9 +34,11 @@ import { useAppSelector } from '@/lib/hooks/useStore' const RightContent = () => { const [open, setOpen] = useState(false) const [openSchema, setOpenSchema] = useState(false) + const [openFileset, setOpenFileset] = useState(false) const searchParams = useSearchParams() const [isShowBtn, setBtnVisible] = useState(true) const [isShowSchemaBtn, setSchemaBtnVisible] = useState(false) + const [isShowFilesetBtn, setFilesetBtnVisible] = useState(false) const store = useAppSelector(state => state.metalakes) const handleCreateCatalog = () => { @@ -46,15 +49,33 @@ const RightContent = () => { setOpenSchema(true) } + const handleCreateFileset = () => { + setOpenFileset(true) + } + useEffect(() => { const paramsSize = [...searchParams.keys()].length const isCatalogList = paramsSize == 1 && searchParams.get('metalake') setBtnVisible(isCatalogList) + const isFilesetList = + paramsSize == 4 && + searchParams.has('metalake') && + searchParams.has('catalog') && + searchParams.get('type') === 'fileset' + searchParams.has('schema') + setFilesetBtnVisible(isFilesetList) + if (store.catalogs.length) { const currentCatalog = store.catalogs.filter(ca => ca.name === searchParams.get('catalog'))[0] - const isHideSchemaAction = ['lakehouse-hudi', 'kafka'].includes(currentCatalog?.provider) && paramsSize == 3 - setSchemaBtnVisible(!isHideSchemaAction && !isCatalogList) + + const isSchemaList = + paramsSize == 3 && + searchParams.has('metalake') && + searchParams.has('catalog') && + searchParams.has('type') && + !['lakehouse-hudi', 'kafka'].includes(currentCatalog?.provider) + setSchemaBtnVisible(isSchemaList) } }, [searchParams, store.catalogs, store.catalogs.length]) @@ -105,6 +126,20 @@ const RightContent = () => { )} + {isShowFilesetBtn && ( + + + + + )} diff --git a/web/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js b/web/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js index cdc94c776df..cf8cc3bafef 100644 --- a/web/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js +++ b/web/web/src/app/metalakes/metalake/rightContent/tabsContent/tableView/TableView.js @@ -41,14 +41,16 @@ import DetailsDrawer from '@/components/DetailsDrawer' import ConfirmDeleteDialog from '@/components/ConfirmDeleteDialog' import CreateCatalogDialog from '../../CreateCatalogDialog' import CreateSchemaDialog from '../../CreateSchemaDialog' +import CreateFilesetDialog from '../../CreateFilesetDialog' import { useAppSelector, useAppDispatch } from '@/lib/hooks/useStore' -import { deleteCatalog, deleteSchema } from '@/lib/store/metalakes' +import { deleteCatalog, deleteFileset, deleteSchema } from '@/lib/store/metalakes' import { to } from '@/lib/utils' import { getCatalogDetailsApi } from '@/lib/api/catalogs' import { getSchemaDetailsApi } from '@/lib/api/schemas' import { useSearchParams } from 'next/navigation' +import { getFilesetDetailsApi } from '@/lib/api/filesets' const fonts = Inconsolata({ subsets: ['latin'] }) @@ -76,6 +78,13 @@ const TableView = () => { const metalake = searchParams.get('metalake') || '' const catalog = searchParams.get('catalog') || '' const type = searchParams.get('type') || '' + const schema = searchParams.get('schema') || '' + + const isKafkaSchema = + paramsSize == 3 && + searchParams.has('metalake') && + searchParams.has('catalog') && + searchParams.get('type') === 'messaging' const defaultPaginationConfig = { pageSize: 10, page: 0 } const pageSizeOptions = [10, 25, 50] @@ -91,6 +100,7 @@ const TableView = () => { const [openConfirmDelete, setOpenConfirmDelete] = useState(false) const [openDialog, setOpenDialog] = useState(false) const [openSchemaDialog, setOpenSchemaDialog] = useState(false) + const [openFilesetDialog, setOpenFilesetDialog] = useState(false) const [dialogData, setDialogData] = useState({}) const [dialogType, setDialogType] = useState('create') const [isHideSchemaEdit, setIsHideSchemaEdit] = useState(true) @@ -463,6 +473,15 @@ const TableView = () => { setOpenDrawer(true) break } + case 'fileset': { + const [err, res] = await to(getFilesetDetailsApi({ metalake, catalog, schema, fileset: row.name })) + if (err || !res) { + throw new Error(err) + } + + setDrawerData(res.fileset) + setOpenDrawer(true) + } default: return } @@ -498,6 +517,18 @@ const TableView = () => { } break } + case 'fileset': { + if (metalake && catalog && schema) { + const [err, res] = await to(getFilesetDetailsApi({ metalake, catalog, schema, fileset: data.row?.name })) + if (err || !res) { + throw new Error(err) + } + + setDialogType('update') + setDialogData(res.fileset) + setOpenFilesetDialog(true) + } + } default: return } @@ -522,6 +553,9 @@ const TableView = () => { case 'schema': dispatch(deleteSchema({ metalake, catalog, type, schema: confirmCacheData.name })) break + case 'fileset': + dispatch(deleteFileset({ metalake, catalog, type, schema, fileset: confirmCacheData.name })) + break default: break } @@ -533,7 +567,12 @@ const TableView = () => { const checkColumns = () => { if ( (paramsSize == 1 && searchParams.has('metalake')) || - (paramsSize == 3 && searchParams.has('metalake') && searchParams.has('catalog') && searchParams.has('type')) + (paramsSize == 3 && searchParams.has('metalake') && searchParams.has('catalog') && searchParams.has('type')) || + (paramsSize == 4 && + searchParams.has('metalake') && + searchParams.has('catalog') && + searchParams.get('type') === 'fileset' && + searchParams.has('schema')) ) { return actionsColumns } else if (paramsSize == 5 && searchParams.has('table')) { @@ -580,6 +619,13 @@ const TableView = () => { + + ) } diff --git a/web/web/src/lib/api/filesets/index.js b/web/web/src/lib/api/filesets/index.js index 81f05488fac..bae492a11de 100644 --- a/web/web/src/lib/api/filesets/index.js +++ b/web/web/src/lib/api/filesets/index.js @@ -27,7 +27,13 @@ const Apis = { GET_DETAIL: ({ metalake, catalog, schema, fileset }) => `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent( catalog - )}/schemas/${encodeURIComponent(schema)}/filesets/${encodeURIComponent(fileset)}` + )}/schemas/${encodeURIComponent(schema)}/filesets/${encodeURIComponent(fileset)}`, + CREATE: ({ metalake, catalog, schema }) => + `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent(catalog)}/schemas/${encodeURIComponent(schema)}/filesets`, + UPDATE: ({ metalake, catalog, schema, fileset }) => + `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent(catalog)}/schemas/${encodeURIComponent(schema)}/filesets/${encodeURIComponent(fileset)}`, + DELETE: ({ metalake, catalog, schema, fileset }) => + `/api/metalakes/${encodeURIComponent(metalake)}/catalogs/${encodeURIComponent(catalog)}/schemas/${encodeURIComponent(schema)}/filesets/${encodeURIComponent(fileset)}` } export const getFilesetsApi = params => { @@ -41,3 +47,15 @@ export const getFilesetDetailsApi = ({ metalake, catalog, schema, fileset }) => url: `${Apis.GET_DETAIL({ metalake, catalog, schema, fileset })}` }) } + +export const createFilesetApi = ({ metalake, catalog, schema, data }) => { + return defHttp.post({ url: `${Apis.CREATE({ metalake, catalog, schema })}`, data }) +} + +export const updateFilesetApi = ({ metalake, catalog, schema, fileset, data }) => { + return defHttp.put({ url: `${Apis.UPDATE({ metalake, catalog, schema, fileset })}`, data }) +} + +export const deleteFilesetApi = ({ metalake, catalog, schema, fileset }) => { + return defHttp.delete({ url: `${Apis.DELETE({ metalake, catalog, schema, fileset })}` }) +} diff --git a/web/web/src/lib/store/metalakes/index.js b/web/web/src/lib/store/metalakes/index.js index 7c58e80e4cc..445d2838d3f 100644 --- a/web/web/src/lib/store/metalakes/index.js +++ b/web/web/src/lib/store/metalakes/index.js @@ -47,7 +47,13 @@ import { deleteSchemaApi } from '@/lib/api/schemas' import { getTablesApi, getTableDetailsApi } from '@/lib/api/tables' -import { getFilesetsApi, getFilesetDetailsApi } from '@/lib/api/filesets' +import { + getFilesetsApi, + getFilesetDetailsApi, + createFilesetApi, + updateFilesetApi, + deleteFilesetApi +} from '@/lib/api/filesets' import { getTopicsApi, getTopicDetailsApi } from '@/lib/api/topics' export const fetchMetalakes = createAsyncThunk('appMetalakes/fetchMetalakes', async (params, { getState }) => { @@ -885,6 +891,67 @@ export const getFilesetDetails = createAsyncThunk( } ) +export const createFileset = createAsyncThunk( + 'appMetalakes/createFileset', + async ({ data, metalake, catalog, type, schema }, { dispatch }) => { + dispatch(setTableLoading(true)) + const [err, res] = await to(createFilesetApi({ data, metalake, catalog, schema })) + dispatch(setTableLoading(false)) + + if (err || !res) { + return { err: true } + } + + const { fileset: filesetItem } = res + + const filesetData = { + ...filesetItem, + node: 'fileset', + id: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${filesetItem.name}}}`, + key: `{{${metalake}}}{{${catalog}}}{{${type}}}{{${schema}}}{{${filesetItem.name}}}`, + path: `?${new URLSearchParams({ metalake, catalog, type, schema, fileset: filesetItem.name }).toString()}`, + name: filesetItem.name, + title: filesetItem.name, + tables: [], + children: [] + } + + dispatch(fetchFilesets({ metalake, catalog, schema, type, init: true })) + + return filesetData + } +) + +export const updateFileset = createAsyncThunk( + 'appMetalakes/updateFileset', + async ({ metalake, catalog, type, schema, fileset, data }, { dispatch }) => { + const [err, res] = await to(updateFilesetApi({ metalake, catalog, schema, fileset, data })) + if (err || !res) { + return { err: true } + } + dispatch(fetchFilesets({ metalake, catalog, type, schema, init: true })) + + return res.catalog + } +) + +export const deleteFileset = createAsyncThunk( + 'appMetalakes/deleteFileset', + async ({ metalake, catalog, type, schema, fileset }, { dispatch }) => { + dispatch(setTableLoading(true)) + const [err, res] = await to(deleteFilesetApi({ metalake, catalog, schema, fileset })) + dispatch(setTableLoading(false)) + + if (err || !res) { + throw new Error(err) + } + + dispatch(fetchFilesets({ metalake, catalog, type, schema, page: 'schemas', init: true })) + + return res + } +) + export const fetchTopics = createAsyncThunk( 'appMetalakes/fetchTopics', async ({ init, page, metalake, catalog, schema }, { getState, dispatch }) => {