feat: search variables page (#3899)

* feat: full text search variables by name, project, table, label, and description * feat: n-gram only variable content field * feat: variables search page (wip) * feat(ui): search variables criteria, link from tables and table pages * feat(ui): added vocabulary search criteria * feat(ui): search from variable annotation * feat(ui): added term descriptions and taxonomy title in variables search page
obiba · Aug 25, 2024 · d275f1d · d275f1d
1 parent bf97ea7
commit d275f1d
Show file tree

Hide file tree

Showing 22 changed files with 526 additions and 57 deletions.
diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/ValuesIndexImpl.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/ValuesIndexImpl.java
@@ -102,13 +102,13 @@ public Document asDocument(VariableEntity entity) {
     Document doc = new Document();
     doc.add(new StringField("tableId", getValueTableReference(), Field.Store.YES));
     doc.add(new StringField("id", entity.getIdentifier(), Field.Store.YES));
+    doc.add(new StringField("type", entity.getType(), Field.Store.YES));
+    doc.add(new StringField("project", table.getDatasource().getName(), Field.Store.YES));
+    doc.add(new StringField("datasource", table.getDatasource().getName(), Field.Store.YES));
+    doc.add(new StringField("table", table.getName(), Field.Store.YES));
 
-    doc.add(new TextField("project", table.getDatasource().getName(), Field.Store.YES));
-    doc.add(new TextField("datasource", table.getDatasource().getName(), Field.Store.YES));
-    doc.add(new TextField("table", table.getName(), Field.Store.YES));
-    doc.add(new TextField("name", entity.getIdentifier(), Field.Store.YES));
-
-    doc.add(new TextField("type", entity.getType(), Field.Store.YES));
+    // tokenized
+    doc.add(new TextField("id-tok", entity.getIdentifier(), Field.Store.YES));
 
     return  doc;
   }

diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesAnalyzer.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesAnalyzer.java
@@ -21,6 +21,7 @@ public class VariablesAnalyzer extends Analyzer {
   private final int maxGram;
 
   public VariablesAnalyzer(int minGram, int maxGram) {
+    super(PER_FIELD_REUSE_STRATEGY);
     this.minGram = minGram;
     this.maxGram = maxGram;
   }
@@ -30,7 +31,8 @@ protected TokenStreamComponents createComponents(String fieldName) {
     src.setMaxTokenLength(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);
     TokenStream tok = new LowerCaseFilter(src);
     tok = new StopFilter(tok, CharArraySet.EMPTY_SET);
-    tok = new NGramTokenFilter(tok, minGram, maxGram, true);
+    if (fieldName.equals("content"))
+      tok = new NGramTokenFilter(tok, minGram, maxGram, true);
     return new TokenStreamComponents(
         r -> {
           src.setMaxTokenLength(StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH);

diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesIndexImpl.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesIndexImpl.java
@@ -50,7 +50,7 @@ public void delete() {
       getIndexFile().delete();
       try (IndexWriter writer = indexManager.newIndexWriter()) {
         // Create a query to match the documents to delete
-        Query query = new TermQuery(new Term("tableId", getValueTableReference()));
+        Query query = new TermQuery(new Term("table-ref", getValueTableReference()));
         // Delete documents that match the query
         writer.deleteDocuments(query);
         writer.commit();
@@ -86,32 +86,35 @@ public void create() {
 
   public Document asDocument(Variable variable) {
     Document doc = new Document();
-    doc.add(new StringField("tableId", getValueTableReference(), Field.Store.YES));
+    doc.add(new StringField("table-ref", getValueTableReference(), Field.Store.YES));
     doc.add(new StringField("id", String.format("%s:%s", getValueTableReference(), variable.getName()), Field.Store.YES));
 
     doc.add(new TextField("project", table.getDatasource().getName(), Field.Store.YES));
     doc.add(new TextField("datasource", table.getDatasource().getName(), Field.Store.YES));
     doc.add(new TextField("table", table.getName(), Field.Store.YES));
     doc.add(new TextField("name", variable.getName(), Field.Store.YES));
 
-    doc.add(new TextField("entityType", variable.getEntityType(), Field.Store.YES));
-    doc.add(new TextField("valueType", variable.getValueType().getName(), Field.Store.YES));
+    doc.add(new TextField("entity-type", variable.getEntityType(), Field.Store.YES));
+    doc.add(new TextField("value-type", variable.getValueType().getName(), Field.Store.YES));
     if (variable.getOccurrenceGroup() != null)
-      doc.add(new TextField("occurrenceGroup", variable.getOccurrenceGroup(), Field.Store.YES));
+      doc.add(new TextField("occurrence-group", variable.getOccurrenceGroup(), Field.Store.YES));
     doc.add(new TextField("repeatable", variable.isRepeatable() + "", Field.Store.YES));
     if (variable.getMimeType() != null)
-      doc.add(new TextField("mimeType", variable.getMimeType(), Field.Store.YES));
+      doc.add(new TextField("mime-type", variable.getMimeType(), Field.Store.YES));
     if (variable.getUnit() != null)
       doc.add(new TextField("unit", variable.getUnit(), Field.Store.YES));
     if (variable.getReferencedEntityType() != null)
-      doc.add(new TextField("referencedEntityType", variable.getReferencedEntityType(), Field.Store.YES));
+      doc.add(new TextField("referenced-entity-type", variable.getReferencedEntityType(), Field.Store.YES));
     doc.add(new TextField("nature", VariableNature.getNature(variable).name(), Field.Store.YES));
 
+    String content = String.format("%s %s %s", table.getDatasource().getName(), table.getName(), variable.getName());
+
     if (variable.hasAttributes()) {
       for (Attribute attribute : variable.getAttributes()) {
         String value = attribute.getValue().toString();
-        if (value != null)
+        if (value != null) {
           doc.add(new TextField(getFieldName(attribute), value, Field.Store.YES));
+        }
       }
     }
 
@@ -128,6 +131,8 @@ public Document asDocument(Variable variable) {
       }
     }
 
+    doc.add(new TextField("content", content, Field.Store.NO));
+
     return doc;
   }
 

diff --git a/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesQueryExecutor.java b/opal-search/src/main/java/org/obiba/opal/search/service/impl/VariablesQueryExecutor.java
@@ -39,7 +39,7 @@ public class VariablesQueryExecutor implements SearchQueryExecutor {
 
   private static final Logger log = LoggerFactory.getLogger(VariablesQueryExecutor.class);
 
-  private final Set<String> DEFAULT_FIELDS = Set.of("name", "project", "table", "valueType", "entityType");
+  private final Set<String> DEFAULT_FIELDS = Set.of("name", "project", "table", "value-type", "entity-type");
 
   private final Directory directory;
 
@@ -56,7 +56,7 @@ public Search.QueryResultDto execute(QuerySettings querySettings) throws SearchE
 
       // Build a QueryParser
       Analyzer analyzer = AnalyzerFactory.newVariablesAnalyzer();
-      QueryParser parser = new QueryParser("name", analyzer);
+      QueryParser parser = new QueryParser("content", analyzer);
 
       // Parse a query (search for books with "Lucene" in the title)
       Query query = parser.parse(querySettings.getQuery());

diff --git a/opal-ui/src/components/charts/CategoricalSummaryChart.vue b/opal-ui/src/components/charts/CategoricalSummaryChart.vue
@@ -115,10 +115,9 @@ const isBar = computed(() => chartType.value === 'bar');
 
 const frequencies = computed(() => {
   if (!props.data) return [];
-  const freqs = props.data.frequencies
-    .filter((f: FrequencyDto) => nonMissingsSelection.value === null || (!nonMissingsSelection.value && f.missing) || (nonMissingsSelection.value && !f.missing));
+  const freqs = props.data.frequencies?.filter((f: FrequencyDto) => nonMissingsSelection.value === null || (!nonMissingsSelection.value && f.missing) || (nonMissingsSelection.value && !f.missing));
 
-  if (props.data.otherFrequency > 0 && nonMissingsSelection.value !== false) {
+  if (props.data.otherFrequency && props.data.otherFrequency > 0 && nonMissingsSelection.value !== false) {
     freqs.push({
       value: t('other'),
       freq: props.data.otherFrequency,

diff --git a/opal-ui/src/components/charts/ContinuousSummaryChart.vue b/opal-ui/src/components/charts/ContinuousSummaryChart.vue
@@ -10,7 +10,7 @@
           {label: $t('density'), value: 'density'},
         ]"
       />
-      <q-badge color="positive" :label="`N:  ${data.summary.n}`"
+      <q-badge color="positive" :label="`N:  ${data.summary?.n || 0}`"
         class="on-right"/>
     </div>
     <div class="row q-mt-md">
@@ -128,7 +128,7 @@ const histoChartData = computed(() => {
   ];
 });
 
-const hasNormal = computed(() => props.data?.summary.min !== props.data?.summary.max);
+const hasNormal = computed(() => props.data?.summary && props.data?.summary.min !== props.data?.summary.max);
 
 const normalLayout = computed(() => {
   if (!props.data) return {};
@@ -151,7 +151,7 @@ const normalLayout = computed(() => {
 });
 
 const normalChartData = computed(() => {
-  if (!props.data) return [];
+  if (!props.data || !props.data.summary) return [];
   const min = props.data.summary.min;
   const max = props.data.summary.max;
   return [

diff --git a/opal-ui/src/components/datasource/AddToViewDialog.vue b/opal-ui/src/components/datasource/AddToViewDialog.vue
@@ -57,7 +57,7 @@
                 <q-select
                   v-else
                   v-model="props.row[ props.col.name ]"
-                  :options="valueTypes"
+                  :options="ValueTypes"
                   dense
                   borderless
                 ></q-select>
@@ -90,7 +90,7 @@ export default defineComponent({
 </script>
 <script setup lang="ts">
 import { AttributeDto, TableDto, VariableDto } from 'src/models/Magma';
-import { valueTypes } from 'src/utils/magma';
+import { ValueTypes } from 'src/utils/magma';
 
 interface DialogProps {
   modelValue: boolean;

diff --git a/opal-ui/src/components/datasource/ContingencyTable.vue b/opal-ui/src/components/datasource/ContingencyTable.vue
@@ -204,7 +204,7 @@ export default defineComponent({
 });
 </script>
 <script setup lang="ts">
-import { valueTypesMap } from 'src/utils/magma';
+import { ValueTypesMap } from 'src/utils/magma';
 import { getLabels } from 'src/utils/attributes';
 import { VariableDto } from 'src/models/Magma';
 import { QueryResultDto } from 'src/models/Search';
@@ -236,7 +236,7 @@ const allVarCatOptions = computed(() => datasourceStore.variables
   .filter((v) => v.categories?.length > 0 || v.valueType === 'boolean')
   .map((v) => ({label: v.name, value: v.name, variable: v })));
 const allVarAltOptions = computed(() => datasourceStore.variables
-  .filter((v) => v.categories?.length > 0 || v.valueType === 'boolean' || valueTypesMap.numerical.includes(v.valueType))
+  .filter((v) => v.categories?.length > 0 || v.valueType === 'boolean' || ValueTypesMap.numerical.includes(v.valueType))
   .map((v) => ({label: v.name, value: v.name, variable: v })));
 
 const variableCat = computed(() => varCat.value?.variable);

diff --git a/opal-ui/src/components/datasource/DatasourceTables.vue b/opal-ui/src/components/datasource/DatasourceTables.vue
@@ -92,6 +92,7 @@
             </q-list>
           </q-btn-dropdown>
           <q-btn v-if="datasourceStore.tables.length && projectsStore.perms.copy?.canCreate()" color="secondary" icon="content_copy" :title="$t('copy')" size="sm" @click="onShowCopy"></q-btn>
+          <q-btn v-if="datasourceStore.tables.length" color="secondary" outline icon="search" :title="$t('search_variables')" size="sm" @click="onSearch"></q-btn>
           <q-btn v-if="datasourceStore.perms.tables?.canDelete()" :disable="removableTables.length === 0" outline color="red" icon="delete" :title="$t('delete')" size="sm" @click="onShowDeleteTables"></q-btn>
         </div>
       </template>
@@ -165,6 +166,7 @@ const route = useRoute();
 const router = useRouter();
 const datasourceStore = useDatasourceStore();
 const projectsStore = useProjectsStore();
+const searchStore = useSearchStore();
 const { t } = useI18n();
 
 const filter = ref('');
@@ -314,4 +316,10 @@ function onShowCopy() {
 function onRestoreViews() {
   showRestoreViews.value = true;
 }
+
+function onSearch() {
+  searchStore.reset();
+  searchStore.variablesQuery.criteria['project'] = [dsName.value];
+  router.push('/search/variables');
+}
 </script>
diff --git a/opal-ui/src/components/datasource/EditVariableDialog.vue b/opal-ui/src/components/datasource/EditVariableDialog.vue
@@ -18,7 +18,7 @@
           />
           <q-select
             v-model="selected.valueType"
-            :options="valueTypes"
+            :options="ValueTypes"
             :label="$t('value_type')"
             :hint="$t('value_type_hint')"
             :disable="editMode && hasValues"
@@ -103,7 +103,7 @@ export default defineComponent({
 <script setup lang="ts">
 import { VariableDto } from 'src/models/Magma';
 import { notifyError } from 'src/utils/notify';
-import { valueTypes } from 'src/utils/magma';
+import { ValueTypes } from 'src/utils/magma';
 
 interface DialogProps {
   modelValue: boolean;

diff --git a/opal-ui/src/components/datasource/TableVariables.vue b/opal-ui/src/components/datasource/TableVariables.vue
@@ -31,6 +31,14 @@
               size="sm"
               @click="onRefresh"
             />
+            <q-btn
+              color="secondary"
+              icon="search"
+              :title="$t('search')"
+              outline
+              size="sm"
+              @click="onSearch"
+            />
             <q-btn
               v-if="datasourceStore.perms.table?.canUpdate()"
               outline
@@ -163,6 +171,7 @@ const router = useRouter();
 const { t } = useI18n();
 const datasourceStore = useDatasourceStore();
 const taxonomiesStore = useTaxonomiesStore();
+const searchStore = useSearchStore();
 const { locale } = useI18n({ useScope: 'global' });
 
 const filter = ref('');
@@ -310,4 +319,11 @@ function onShowDeleteAnnotation() {
   annotationOperation.value = 'delete';
   showAnnotate.value = true;
 }
+
+function onSearch() {
+  searchStore.reset();
+  searchStore.variablesQuery.criteria['project'] = [dsName.value];
+  searchStore.variablesQuery.criteria['table'] = [tName.value];
+  router.push('/search/variables');
+}
 </script>
diff --git a/opal-ui/src/components/datasource/VariableAttributes.vue b/opal-ui/src/components/datasource/VariableAttributes.vue
@@ -37,6 +37,17 @@
             <q-item-section v-if="canUpdate" side>
               <table>
                 <tr>
+                  <td>
+                    <q-btn
+                      rounded
+                      dense
+                      flat
+                      size="sm"
+                      color="secondary"
+                      :title="$t('search')"
+                      icon="search"
+                      @click="onSearch(annotation)" />
+                  </td>
                   <td>
                     <q-btn
                       rounded
@@ -201,8 +212,10 @@ import ConfirmDialog from 'src/components/ConfirmDialog.vue';
 import { AttributeDto } from 'src/models/Magma';
 
 const { t } = useI18n();
+const router = useRouter();
 const datasourceStore = useDatasourceStore();
 const taxonomiesStore = useTaxonomiesStore();
+const searchStore = useSearchStore();
 
 const tab = ref('annotations');
 const tableRef = ref();
@@ -255,6 +268,16 @@ const bundles = computed(() => datasourceStore.variableAttributesBundles || []);
 const labelBundle = computed(() => bundles.value.find((bndl) => bndl.id === 'label'));
 const descriptionBundle = computed(() => bundles.value.find((bndl) => bndl.id === 'description'));
 
+function onSearch(annotation: Annotation | undefined) {
+  if (annotation) {
+    searchStore.reset();
+    searchStore.variablesQuery.criteria['project'] = [datasourceStore.datasource.name];
+    searchStore.variablesQuery.criteria['table'] = [datasourceStore.table.name];
+    searchStore.variablesQuery.criteria[`${annotation.taxonomy.name}-${annotation.vocabulary.name}`] = [annotation.term.name];
+    router.push('/search/variables');
+  }
+}
+
 function onShowAnnotate(annotation: Annotation | undefined) {
   annotationSelected.value = annotation;
   showAnnotate.value = true;

diff --git a/opal-ui/src/components/datasource/VariableScriptDialog.vue b/opal-ui/src/components/datasource/VariableScriptDialog.vue
@@ -16,7 +16,7 @@
           <div class="row q-gutter-md q-mb-md">
             <q-select
               v-model="valueType"
-              :options="valueTypes"
+              :options="ValueTypes"
               :label="$t('value_type')"
               dense
               style="width: 200px;"/>
@@ -70,7 +70,7 @@ export default defineComponent({
 </script>
 <script setup lang="ts">
 import { VAceEditor } from 'vue3-ace-editor';
-import { valueTypes } from 'src/utils/magma';
+import { ValueTypes } from 'src/utils/magma';
 import { VariableDto, AttributeDto } from 'src/models/Magma';
 
 interface DialogProps {

diff --git a/opal-ui/src/components/datasource/import/ImportCsvForm.vue b/opal-ui/src/components/datasource/import/ImportCsvForm.vue
@@ -29,7 +29,7 @@
       <div class="col">
         <q-select
           v-model="defaultValueType"
-          :options="valueTypes"
+          :options="ValueTypes"
           :label="$t('default_value_type')"
           @update:model-value="onUpdate"
           dense/>
@@ -88,7 +88,7 @@ export default defineComponent({
 <script setup lang="ts">
 import { DatasourceFactory } from 'src/components/models';
 import { FileDto } from 'src/models/Opal';
-import { valueTypes } from 'src/utils/magma';
+import { ValueTypes } from 'src/utils/magma';
 import FileSelect from 'src/components/files/FileSelect.vue';
 
 interface ImportCsvFormProps {

diff --git a/opal-ui/src/i18n/en/index.js b/opal-ui/src/i18n/en/index.js
@@ -1066,11 +1066,18 @@ export default {
   variables_index_cleared: 'Variables index cleared.',
   variables_index_clear_error: 'Variables index clear error.',
   variables_search: 'Variables search',
+  variables_search_info: 'Search variables using controlled vocabularies and full-text search. The controlled vocabularies are defined by the taxonomies. The full-text search applies to the variable name and labels. See [Search Variables documentation](http://opaldoc.obiba.org/en/latest/web-user-guide/search/variables.html) for more details.',
   variables_search_caption: 'Search for variables by name, label, description, category, attribute or vocabulary term.',
   entity: 'Entity',
   entity_search: 'Entity search',
   entity_search_info: 'Search an entity by providing its type and its identifier. See [Search Entity documentation](http://opaldoc.obiba.org/en/latest/web-user-guide/search/entity.html) for more details.',
   entity_search_caption: 'Search for entity by identifier.',
+  advanced_search: 'Advanced search',
+  more_results: 'More results',
+  filters: 'Filters',
+  filter: 'Filter',
+  nature: 'Nature',
+  results: 'Results',
   stats: {
     min: 'Minimun',
     max: 'Maximum',
@@ -1095,5 +1102,13 @@ export default {
       UPTODATE: 'Up to date',
       UNRECOGNIZED: 'Unrecognized'
     }
+  },
+  variable_nature: {
+    CATEGORICAL: 'Categorical',
+    CONTINUOUS: 'Continuous',
+    TEMPORAL: 'Temporal',
+    GEO: 'Geographical',
+    BINARY: 'Binary',
+    UNDETERMINED: 'Undetermined',
   }
 };