From d275f1dbfbc7d7db7f414937b0e071e481f6c487 Mon Sep 17 00:00:00 2001 From: Yannick Marcon Date: Sun, 25 Aug 2024 15:22:47 +0200 Subject: [PATCH] feat: search variables page (#3899) * feat: full text search variables by name, project, table, label, and description * feat: n-gram only variable content field * feat: variables search page (wip) * feat(ui): search variables criteria, link from tables and table pages * feat(ui): added vocabulary search criteria * feat(ui): search from variable annotation * feat(ui): added term descriptions and taxonomy title in variables search page --- .../search/service/impl/ValuesIndexImpl.java | 12 +- .../service/impl/VariablesAnalyzer.java | 4 +- .../service/impl/VariablesIndexImpl.java | 21 +- .../service/impl/VariablesQueryExecutor.java | 4 +- .../charts/CategoricalSummaryChart.vue | 5 +- .../charts/ContinuousSummaryChart.vue | 6 +- .../components/datasource/AddToViewDialog.vue | 4 +- .../datasource/ContingencyTable.vue | 4 +- .../datasource/DatasourceTables.vue | 8 + .../datasource/EditVariableDialog.vue | 4 +- .../components/datasource/TableVariables.vue | 16 + .../datasource/VariableAttributes.vue | 23 ++ .../datasource/VariableScriptDialog.vue | 4 +- .../datasource/import/ImportCsvForm.vue | 4 +- opal-ui/src/i18n/en/index.js | 15 + opal-ui/src/i18n/fr/index.js | 17 +- opal-ui/src/layouts/MainLayout.vue | 34 +- opal-ui/src/pages/AdminSearchPage.vue | 2 +- opal-ui/src/pages/SearchVariablesPage.vue | 299 ++++++++++++++++++ opal-ui/src/pages/SigninPage.vue | 2 + opal-ui/src/stores/search.ts | 82 +++++ opal-ui/src/utils/magma.ts | 13 +- 22 files changed, 526 insertions(+), 57 deletions(-) diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/ValuesIndexImpl.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/ValuesIndexImpl.java index 780bc88d26..c55112d8b7 100644 --- a/opal-search/src/main/java/org/obiba/opal/search/service/impl/ValuesIndexImpl.java +++ b/opal-search/src/main/java/org/obiba/opal/search/service/impl/ValuesIndexImpl.java @@ -102,13 +102,13 @@ public Document asDocument(VariableEntity entity) { Document doc = new Document(); doc.add(new StringField("tableId", getValueTableReference(), Field.Store.YES)); doc.add(new StringField("id", entity.getIdentifier(), Field.Store.YES)); + doc.add(new StringField("type", entity.getType(), Field.Store.YES)); + doc.add(new StringField("project", table.getDatasource().getName(), Field.Store.YES)); + doc.add(new StringField("datasource", table.getDatasource().getName(), Field.Store.YES)); + doc.add(new StringField("table", table.getName(), Field.Store.YES)); - doc.add(new TextField("project", table.getDatasource().getName(), Field.Store.YES)); - doc.add(new TextField("datasource", table.getDatasource().getName(), Field.Store.YES)); - doc.add(new TextField("table", table.getName(), Field.Store.YES)); - doc.add(new TextField("name", entity.getIdentifier(), Field.Store.YES)); - - doc.add(new TextField("type", entity.getType(), Field.Store.YES)); + // tokenized + doc.add(new TextField("id-tok", entity.getIdentifier(), Field.Store.YES)); return doc; } diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesAnalyzer.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesAnalyzer.java index 862e6790eb..e525b2bf5f 100644 --- a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesAnalyzer.java +++ b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesAnalyzer.java @@ -21,6 +21,7 @@ public class VariablesAnalyzer extends Analyzer { private final int maxGram; public VariablesAnalyzer(int minGram, int maxGram) { + super(PER_FIELD_REUSE_STRATEGY); this.minGram = minGram; this.maxGram = maxGram; } @@ -30,7 +31,8 @@ protected TokenStreamComponents createComponents(String fieldName) { src.setMaxTokenLength(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH); TokenStream tok = new LowerCaseFilter(src); tok = new StopFilter(tok, CharArraySet.EMPTY_SET); - tok = new NGramTokenFilter(tok, minGram, maxGram, true); + if (fieldName.equals("content")) + tok = new NGramTokenFilter(tok, minGram, maxGram, true); return new TokenStreamComponents( r -> { src.setMaxTokenLength(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH); diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesIndexImpl.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesIndexImpl.java index 52d1a92aac..d5be9a1514 100644 --- a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesIndexImpl.java +++ b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesIndexImpl.java @@ -50,7 +50,7 @@ public void delete() { getIndexFile().delete(); try (IndexWriter writer = indexManager.newIndexWriter()) { // Create a query to match the documents to delete - Query query = new TermQuery(new Term("tableId", getValueTableReference())); + Query query = new TermQuery(new Term("table-ref", getValueTableReference())); // Delete documents that match the query writer.deleteDocuments(query); writer.commit(); @@ -86,7 +86,7 @@ public void create() { public Document asDocument(Variable variable) { Document doc = new Document(); - doc.add(new StringField("tableId", getValueTableReference(), Field.Store.YES)); + doc.add(new StringField("table-ref", getValueTableReference(), Field.Store.YES)); doc.add(new StringField("id", String.format("%s:%s", getValueTableReference(), variable.getName()), Field.Store.YES)); doc.add(new TextField("project", table.getDatasource().getName(), Field.Store.YES)); @@ -94,24 +94,27 @@ public Document asDocument(Variable variable) { doc.add(new TextField("table", table.getName(), Field.Store.YES)); doc.add(new TextField("name", variable.getName(), Field.Store.YES)); - doc.add(new TextField("entityType", variable.getEntityType(), Field.Store.YES)); - doc.add(new TextField("valueType", variable.getValueType().getName(), Field.Store.YES)); + doc.add(new TextField("entity-type", variable.getEntityType(), Field.Store.YES)); + doc.add(new TextField("value-type", variable.getValueType().getName(), Field.Store.YES)); if (variable.getOccurrenceGroup() != null) - doc.add(new TextField("occurrenceGroup", variable.getOccurrenceGroup(), Field.Store.YES)); + doc.add(new TextField("occurrence-group", variable.getOccurrenceGroup(), Field.Store.YES)); doc.add(new TextField("repeatable", variable.isRepeatable() + "", Field.Store.YES)); if (variable.getMimeType() != null) - doc.add(new TextField("mimeType", variable.getMimeType(), Field.Store.YES)); + doc.add(new TextField("mime-type", variable.getMimeType(), Field.Store.YES)); if (variable.getUnit() != null) doc.add(new TextField("unit", variable.getUnit(), Field.Store.YES)); if (variable.getReferencedEntityType() != null) - doc.add(new TextField("referencedEntityType", variable.getReferencedEntityType(), Field.Store.YES)); + doc.add(new TextField("referenced-entity-type", variable.getReferencedEntityType(), Field.Store.YES)); doc.add(new TextField("nature", VariableNature.getNature(variable).name(), Field.Store.YES)); + String content = String.format("%s %s %s", table.getDatasource().getName(), table.getName(), variable.getName()); + if (variable.hasAttributes()) { for (Attribute attribute : variable.getAttributes()) { String value = attribute.getValue().toString(); - if (value != null) + if (value != null) { doc.add(new TextField(getFieldName(attribute), value, Field.Store.YES)); + } } } @@ -128,6 +131,8 @@ public Document asDocument(Variable variable) { } } + doc.add(new TextField("content", content, Field.Store.NO)); + return doc; } diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesQueryExecutor.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesQueryExecutor.java index ba10489bf9..823375a6a5 100644 --- a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesQueryExecutor.java +++ b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesQueryExecutor.java @@ -39,7 +39,7 @@ public class VariablesQueryExecutor implements SearchQueryExecutor { private static final Logger log = LoggerFactory.getLogger(VariablesQueryExecutor.class); - private final Set DEFAULT_FIELDS = Set.of("name", "project", "table", "valueType", "entityType"); + private final Set DEFAULT_FIELDS = Set.of("name", "project", "table", "value-type", "entity-type"); private final Directory directory; @@ -56,7 +56,7 @@ public Search.QueryResultDto execute(QuerySettings querySettings) throws SearchE // Build a QueryParser Analyzer analyzer = AnalyzerFactory.newVariablesAnalyzer(); - QueryParser parser = new QueryParser("name", analyzer); + QueryParser parser = new QueryParser("content", analyzer); // Parse a query (search for books with "Lucene" in the title) Query query = parser.parse(querySettings.getQuery()); diff --git a/opal-ui/src/components/charts/CategoricalSummaryChart.vue b/opal-ui/src/components/charts/CategoricalSummaryChart.vue index b21c57353b..3f0660335b 100644 --- a/opal-ui/src/components/charts/CategoricalSummaryChart.vue +++ b/opal-ui/src/components/charts/CategoricalSummaryChart.vue @@ -115,10 +115,9 @@ const isBar = computed(() => chartType.value === 'bar'); const frequencies = computed(() => { if (!props.data) return []; - const freqs = props.data.frequencies - .filter((f: FrequencyDto) => nonMissingsSelection.value === null || (!nonMissingsSelection.value && f.missing) || (nonMissingsSelection.value && !f.missing)); + const freqs = props.data.frequencies?.filter((f: FrequencyDto) => nonMissingsSelection.value === null || (!nonMissingsSelection.value && f.missing) || (nonMissingsSelection.value && !f.missing)); - if (props.data.otherFrequency > 0 && nonMissingsSelection.value !== false) { + if (props.data.otherFrequency && props.data.otherFrequency > 0 && nonMissingsSelection.value !== false) { freqs.push({ value: t('other'), freq: props.data.otherFrequency, diff --git a/opal-ui/src/components/charts/ContinuousSummaryChart.vue b/opal-ui/src/components/charts/ContinuousSummaryChart.vue index 6b8aa2a74d..a0e2b58ea4 100644 --- a/opal-ui/src/components/charts/ContinuousSummaryChart.vue +++ b/opal-ui/src/components/charts/ContinuousSummaryChart.vue @@ -10,7 +10,7 @@ {label: $t('density'), value: 'density'}, ]" /> -
@@ -128,7 +128,7 @@ const histoChartData = computed(() => { ]; }); -const hasNormal = computed(() => props.data?.summary.min !== props.data?.summary.max); +const hasNormal = computed(() => props.data?.summary && props.data?.summary.min !== props.data?.summary.max); const normalLayout = computed(() => { if (!props.data) return {}; @@ -151,7 +151,7 @@ const normalLayout = computed(() => { }); const normalChartData = computed(() => { - if (!props.data) return []; + if (!props.data || !props.data.summary) return []; const min = props.data.summary.min; const max = props.data.summary.max; return [ diff --git a/opal-ui/src/components/datasource/AddToViewDialog.vue b/opal-ui/src/components/datasource/AddToViewDialog.vue index 0beffe2e2f..00b6ef1402 100644 --- a/opal-ui/src/components/datasource/AddToViewDialog.vue +++ b/opal-ui/src/components/datasource/AddToViewDialog.vue @@ -57,7 +57,7 @@ @@ -90,7 +90,7 @@ export default defineComponent({ diff --git a/opal-ui/src/components/datasource/EditVariableDialog.vue b/opal-ui/src/components/datasource/EditVariableDialog.vue index 7c19f63916..242e6e732b 100644 --- a/opal-ui/src/components/datasource/EditVariableDialog.vue +++ b/opal-ui/src/components/datasource/EditVariableDialog.vue @@ -18,7 +18,7 @@ /> import { VariableDto } from 'src/models/Magma'; import { notifyError } from 'src/utils/notify'; -import { valueTypes } from 'src/utils/magma'; +import { ValueTypes } from 'src/utils/magma'; interface DialogProps { modelValue: boolean; diff --git a/opal-ui/src/components/datasource/TableVariables.vue b/opal-ui/src/components/datasource/TableVariables.vue index a9730b81db..ab18b6d61c 100644 --- a/opal-ui/src/components/datasource/TableVariables.vue +++ b/opal-ui/src/components/datasource/TableVariables.vue @@ -31,6 +31,14 @@ size="sm" @click="onRefresh" /> + diff --git a/opal-ui/src/components/datasource/VariableAttributes.vue b/opal-ui/src/components/datasource/VariableAttributes.vue index 288f6ad0ca..490b5483f4 100644 --- a/opal-ui/src/components/datasource/VariableAttributes.vue +++ b/opal-ui/src/components/datasource/VariableAttributes.vue @@ -37,6 +37,17 @@ +
+ + datasourceStore.variableAttributesBundles || []); const labelBundle = computed(() => bundles.value.find((bndl) => bndl.id === 'label')); const descriptionBundle = computed(() => bundles.value.find((bndl) => bndl.id === 'description')); +function onSearch(annotation: Annotation | undefined) { + if (annotation) { + searchStore.reset(); + searchStore.variablesQuery.criteria['project'] = [datasourceStore.datasource.name]; + searchStore.variablesQuery.criteria['table'] = [datasourceStore.table.name]; + searchStore.variablesQuery.criteria[`${annotation.taxonomy.name}-${annotation.vocabulary.name}`] = [annotation.term.name]; + router.push('/search/variables'); + } +} + function onShowAnnotate(annotation: Annotation | undefined) { annotationSelected.value = annotation; showAnnotate.value = true; diff --git a/opal-ui/src/components/datasource/VariableScriptDialog.vue b/opal-ui/src/components/datasource/VariableScriptDialog.vue index 8ffc9361dd..7e51669678 100644 --- a/opal-ui/src/components/datasource/VariableScriptDialog.vue +++ b/opal-ui/src/components/datasource/VariableScriptDialog.vue @@ -16,7 +16,7 @@
@@ -70,7 +70,7 @@ export default defineComponent({ diff --git a/opal-ui/src/pages/SigninPage.vue b/opal-ui/src/pages/SigninPage.vue index c449970a56..8334b88852 100644 --- a/opal-ui/src/pages/SigninPage.vue +++ b/opal-ui/src/pages/SigninPage.vue @@ -163,6 +163,7 @@ const identityProvidersStore = useIdentityProvidersStore(); const tokensStore = useTokensStore(); const identifiersStore = useIdentifiersStore(); const appsStore = useAppsStore(); +const searchStore = useSearchStore(); const { cookies } = useCookies(); const { locale, t } = useI18n({ useScope: 'global' }); @@ -212,6 +213,7 @@ onMounted(() => { }); identifiersStore.reset(); appsStore.reset(); + searchStore.reset(); }); function onLocaleSelection(localeOpt: { label: string; value: string }) { diff --git a/opal-ui/src/stores/search.ts b/opal-ui/src/stores/search.ts index 27e9e7016c..c3f5b17e34 100644 --- a/opal-ui/src/stores/search.ts +++ b/opal-ui/src/stores/search.ts @@ -1,8 +1,52 @@ import { defineStore } from 'pinia'; import { api } from 'src/boot/api'; +import { ItemFieldsDto, ItemResultDto } from 'src/models/Search'; + +export interface ItemFieldsResultDto extends ItemResultDto { + 'Search.ItemFieldsDto.item': ItemFieldsDto; +} + +export interface SearchQuery { + query: string; + criteria: SearchCriteria; +} + +export interface SearchCriteria { + [key: string]: string[]; +} export const useSearchStore = defineStore('search', () => { + const variablesQuery = ref({ + query: '', + criteria: { + project: [], + table: [], + } as SearchCriteria, + }); + + function reset() { + variablesQuery.value = { + query: '', + criteria: { + project: [], + table: [], + } as SearchCriteria + }; + } + + async function searchVariables(limit: number) { + let fullQuery = variablesQuery.value.query?.trim() || ''; + Object.keys(variablesQuery.value.criteria).forEach((key) => { + const terms = variablesQuery.value.criteria[key]; + if (terms.length > 0) { + const statement = `(${terms.map((t) => `${key}:"${t}"`).join(' OR ')})` + fullQuery = fullQuery.length === 0 ? statement : `${fullQuery} AND ${statement}`; + } + }); + return search(fullQuery, limit, ['label', 'label-en']); + } + async function search(query: string, limit: number, fields: string[] | undefined) { return api.get('/datasources/variables/_search', { params: { query, limit, field: fields }, @@ -22,10 +66,48 @@ export const useSearchStore = defineStore('search', () => { .then(response => response.data); } + async function getTables() { + return api.get('/datasources/tables') + .then(response => response.data); + } + + // + // Results helpers + // + + function getLabels(item: ItemFieldsResultDto) { + const fields = item['Search.ItemFieldsDto.item'].fields; + if (!fields) { + return []; + } + const labels = []; + for (const field of fields) { + if (field.key.startsWith('label')) { + const tokens = field.key.split('-'); + labels.push({ value: field.value, locale: tokens.length > 1 ? tokens[1] : undefined }); + } + } + return labels; + } + + function getField(item: ItemFieldsResultDto, key: string) { + const fields = item['Search.ItemFieldsDto.item'].fields; + if (!fields) { + return ''; + } + return fields.find((field) => field.key === key)?.value; + } + return { + variablesQuery, + reset, search, + searchVariables, clearIndex, getEntityTables, + getTables, + getLabels, + getField, }; }); diff --git a/opal-ui/src/utils/magma.ts b/opal-ui/src/utils/magma.ts index a2125bf9b9..9215614206 100644 --- a/opal-ui/src/utils/magma.ts +++ b/opal-ui/src/utils/magma.ts @@ -1,4 +1,4 @@ -export const valueTypes = [ +export const ValueTypes = [ 'text', 'integer', 'decimal', @@ -12,10 +12,19 @@ export const valueTypes = [ 'locale', ]; -export const valueTypesMap = { +export const ValueTypesMap = { textual: ['text', 'locale'], numerical: ['integer', 'decimal'], temporal: ['date', 'datetime'], geospatial: ['point', 'linestring', 'polygon'], other: ['binary', 'boolean'], }; + +export const VariableNatures = { + CATEGORICAL: 'CATEGORICAL', + CONTINUOUS: 'CONTINUOUS', + TEMPORAL: 'TEMPORAL', + GEO: 'GEO', + BINARY: 'BINARY', + UNDETERMINED: 'UNDETERMINED', +};