Templates admin UI

.dockerignore

@@ -1,4 +1,5 @@
 *.vcf.gz
 ri_tools/vcf/files_to_read/*vcf.gz
 beacon/connections/mongo/data/datasets/*
-beacon/connections/mongo/data/db/*
+beacon/connections/mongo/data/db/*
+*.json.gz

.gitignore

@@ -8,4 +8,5 @@ beacon/connections/mongo/data/rd-connect
 *.vcf.gz
 beacon/connections/mongo/data/db/*
 !beacon/connections/mongo/data/db/dir.txt
-*.sqlite3
+*.sqlite3
+*.json.gz

adminui/adminclient/views.py

@@ -74,28 +74,150 @@ def default_view(request):
     return render(request, template, context)
 
 def entry_types(request):
-    entry_types_list=['analysis', 'biosample', 'cohort', 'dataset', 'genomicVariant', 'individual', 'run']
     form =EntryTypesForm()
     context = {'form': form}
     if request.method == 'POST':
         form = EntryTypesForm(request.POST)
         if form.is_valid():
-            entryTypes = form.cleaned_data['EntryTypes']
-            for entry_type in entry_types_list:
-                if entry_type not in entryTypes:
-                    with open("adminui/beacon/conf/" + entry_type + ".py") as f:
-                        lines = f.readlines()
-                    with open("adminui/beacon/conf/"+ entry_type + ".py", "w") as f:
-                        new_lines =''
-                        for line in lines:
-                            if 'endpoint_name' in str(line):
-                                new_lines+=""+"\n"
+            analysis=form.cleaned_data['Analysis']
+            analysis_endpoint_name = form.cleaned_data['AnalysisEndpointName']
+            biosample=form.cleaned_data['Biosample']
+            biosample_endpoint_name = form.cleaned_data['BiosampleEndpointName']
+            cohort=form.cleaned_data['Cohort']
+            cohort_endpoint_name = form.cleaned_data['CohortEndpointName']
+            dataset=form.cleaned_data['Dataset']
+            dataset_endpoint_name = form.cleaned_data['DatasetEndpointName']
+            genomicVariant=form.cleaned_data['GenomicVariant']
+            genomicVariant_endpoint_name = form.cleaned_data['GenomicVariantEndpointName']
+            individual=form.cleaned_data['Individual']
+            individual_endpoint_name = form.cleaned_data['IndividualEndpointName']
+            run=form.cleaned_data['Run']
+            run_endpoint_name = form.cleaned_data['RunEndpointName']
+            if analysis != None:
+                analysis_endpoints=form.cleaned_data['AnalysisEndpoints']
+                analysis_non_filtered = form.cleaned_data['AnalysisNonFiltered']
+                with open("adminui/beacon/conf/" + 'analysis' + ".py") as f:
+                    lines = f.readlines()
+                with open("adminui/beacon/conf/"+ 'analysis' + ".py", "w") as f:
+                    new_lines =''
+                    for line in lines:
+                        if 'endpoint_name' in str(line):
+                            new_lines+="endpoint_name="+'"'+analysis_endpoint_name+'"'+"\n"
+                        elif 'allow_queries_without_filters' in str(line):
+                            if "True" in str(line):
+                                new_line=line.replace("True",str(analysis_non_filtered))
+                                new_lines+=new_line
+                            elif "False" in str(line):
+                                new_line=line.replace("False",str(analysis_non_filtered))
+                                new_lines+=new_line
+                        elif 'singleEntryUrl' in str(line):
+                            endpoint_to_look=analysis_endpoint_name + '/{id}'
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
                             else:
-                                new_lines+=line
-
-                        f.write(new_lines)
-                    f.close()
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        elif 'biosample_lookup' in str(line):
+                            endpoint_to_look=analysis_endpoint_name + '/{id}/'+biosample_endpoint_name
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                            else:
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        elif 'cohort_lookup' in str(line):
+                            endpoint_to_look=analysis_endpoint_name + '/{id}/'+cohort_endpoint_name
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                            else:
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        elif 'dataset_lookup' in str(line):
+                            endpoint_to_look = analysis_endpoint_name + '/{id}/'+dataset_endpoint_name
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                            else:
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        elif 'genomicVariant_lookup' in str(line):
+                            endpoint_to_look=analysis_endpoint_name + '/{id}/'+genomicVariant_endpoint_name
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                            else:
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        elif 'individual_lookup' in str(line):
+                            endpoint_to_look=analysis_endpoint_name + '/{id}/'+individual_endpoint_name
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                            else:
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        elif 'run_lookup' in str(line):
+                            endpoint_to_look=analysis_endpoint_name + '/{id}/'+run_endpoint_name
+                            if endpoint_to_look in analysis_endpoints:
+                                if 'False' in str(line):
+                                    new_line=line.replace("False","True")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                            else:
+                                if 'True' in str(line):
+                                    new_line=line.replace("True","False")
+                                    new_lines+=new_line
+                                else:
+                                    new_lines+=line
+                        else:
+                            new_lines+=line
+                    f.write(new_lines)
+                f.close()
+
 
+
             return redirect("adminclient:entry_types")
+        else:
+            context = {'form': form}
+
     template = "general_configuration/entry_types.html"
     return render(request, template, context)

adminui/templates/general_configuration/entry_types.html

@@ -18,9 +18,14 @@
       <form method="post" onsubmit="openLoader()">
         {% csrf_token %}
           <div>
+
             {% for f in form %}
-            <p>{{f.help_text|safe}} {{f}} </p>
-            {% endfor %}
+            {% if f.errors %}
+              <p>{{f.help_text|safe}} {{f}} {{f.errors|escape}}</p>
+            {% else %}
+            <p>{{f.help_text|safe}} {{f}}</p>
+            {% endif %}
+            {% endfor %}'
             <span id="actives-label"></span>
           </div>
           <div>

beacon/conf/analysis.py

@@ -1,4 +1,4 @@
-endpoint_name='analyses' # Leave it blank ('') to deactivate the endpoint.
+endpoint_name="analyses"
 open_api_endpoints_definition='https://raw.githubusercontent.com/ga4gh-beacon/beacon-v2/main/models/json/beacon-v2-default-model/analyses/endpoints.json'
 database='mongo' # The name must match the folder's name in connection that belongs to the desired database.
 

beacon/conf/api_version.yml

@@ -1 +1 @@
-api_version: v2.0-46ff62d
+api_version: v2.0-ccd8b4e

beacon/connections/mongo/filters.py

@@ -830,7 +830,7 @@ def apply_alphanumeric_filter(self, query: dict, filter: AlphanumericFilter, col
                 elif len(filter.value)==2:
                     dict_regex['$regex']='^NC_0000'+filter.value+'.'+'10:g'+'|'+'^NC_0000'+filter.value+'.'+'11:g'+'|'+'^NC_0000'+filter.value+'.'+'9:g'
                 else:
-                    dict_regex['$regex']='^NC_00000'+filter.value+'.'+'10:g'+'|'+'^NC_00000'+filter.value+'.'+'11:g'+'|'+'^NC_00000'+filter.value+'.'+'9:g'
+                    dict_regex['$regex']='^NC_0000'+filter.value+'.'+'10:g'+'|'+'^NC_0000'+filter.value+'.'+'11:g'+'|'+'^NC_0000'+filter.value+'.'+'9:g'+'|'+'^NC_00000'+filter.value+'.'+'10:g'+'|'+'^NC_00000'+filter.value+'.'+'11:g'+'|'+'^NC_00000'+filter.value+'.'+'9:g'
             elif '>' in filter.value:# pragma: no cover
                 newvalue=filter.value.replace(">",">")
                 dict_regex=newvalue

beacon/connections/mongo/reindex.py

@@ -32,25 +32,15 @@
     client[dbname].validate_collection("similarities")
 except Exception:
     db=client[dbname].create_collection(name="similarities")
-#client[dbname].analyses.create_index([("$**", "text")])
-#client[dbname].biosamples.create_index([("$**", "text")])
-#client[dbname].cohorts.create_index([("$**", "text")])
-#client[dbname].datasets.create_index([("$**", "text")])
-#genomicVariations.create_index([("$**", "text")])
-#genomicVariations.create_index([("caseLevelData.biosampleId", 1)])
-genomicVariations.create_index([("variation.location.interval.start.value", 1),("variation.location.interval.end.value", 1)])
-genomicVariations.create_index([("variation.alternateBases", 1),("variation.referenceBases", 1),("variation.location.interval.start.value", 1), ("variation.location.interval.end.value", 1)])
-genomicVariations.create_index([("datasetId", 1)])
-#genomicVariations.create_index([("variantInternalId", 1)])
+
+genomicVariations.create_index([("variation.location.interval.start.value", 1),("variation.location.interval.end.value", 1)]) 
+genomicVariations.create_index([("variation.alternateBases", 1),("variation.referenceBases", 1),("variation.location.interval.start.value", 1), ("variation.location.interval.end.value", 1)]) # for sequence queries
+genomicVariations.create_index([("datasetId", 1)]) # splits all the docs into datasets faster
+genomicVariations.create_index([("variantInternalId", 1)]) # enables the g_variants/{id}/endpoint query to do it faster
 #genomicVariations.create_index([("variation.location.interval.start.value", 1)])
-#genomicVariations.create_index([("variation.location.interval.end.value", 1)])
+genomicVariations.create_index([("variation.location.interval.end.value", 1)])
 genomicVariations.create_index([("identifiers.genomicHGVSId", 1)])
-#genomicVariations.create_index([("datasetId", 1), ("variation.location.interval.start.value", 1), ("variation.referenceBases", 1), ("variation.alternateBases", 1)])
 genomicVariations.create_index([("molecularAttributes.geneIds", 1), ("variation.variantType", 1)])
 caseLevelData.create_index([("id", 1), ("datasetId", 1)])
 caseLevelData.create_index([("datasetId", 1)])
-#client[dbname].individuals.create_index([("$**", "text")])
-#client[dbname].runs.create_index([("$**", "text")])
-#collection_name = client[dbname].analyses
-#print(collection_name.index_information())
 

beacon/connections/mongo/request_parameters.py

@@ -90,10 +90,16 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
     subquery["$and"] = []
     subqueryor={}
     subqueryor["$or"] = []
+    referencedict={}
+    referencedict["$or"] = []
     startquery={}
     startquery["$and"]=[]
+    startrangequery={}
+    startrangequery["$and"]=[]
     endquery={}
     endquery["$and"]=[]
+    length_query={}
+    length_query["$and"]=[]
     equal=False
     isBracket=False
     for k, v in qparams.query.requestParameters.items():
@@ -121,15 +127,29 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
                     startquery["$and"].append(startdictvalue)
             else:
                 startvalue=v
-                if isinstance(endvalue, list):
-                    finalvalue=str(0)+','+str(endvalue[0])
+                if isinstance(startvalue, list):
+                    finalvalue=str(0)+','+str(startvalue[0]-1)
                 else:
-                    finalvalue=str(0)+','+str(endvalue)
+                    finalvalue=str(0)+','+str(startvalue-1)
                 finalvalue = finalvalue.split(',')
                 filters = generate_position_filter_start(self, k, finalvalue)
                 for filter in filters:
                     startdictvalue=apply_alphanumeric_filter(self, {}, filter, collection, dataset, True)
-                    startquery["$and"].append(startdictvalue)            
+                    length_query["$and"].append(startdictvalue)
+                if isinstance(endvalue, list) and isinstance(startvalue, list):
+                    finalvalue=str(startvalue[0])+','+str(endvalue[0])
+                if isinstance(startvalue, list):
+                    finalvalue=str(startvalue[0])+','+str(endvalue[0])
+                elif isinstance(endvalue, list):
+                    finalvalue=str(startvalue)+','+str(endvalue[0])
+                else:
+                    finalvalue=str(startvalue)+','+str(endvalue)
+                finalvalue = finalvalue.split(',')
+                filters = generate_position_filter_start(self, k, finalvalue)
+                for filter in filters:
+                    startdict=apply_alphanumeric_filter(self, {}, filter, collection, dataset, True)
+                    startrangequery["$and"].append(startdict)  
+                subqueryor["$or"].append(startrangequery)            
         elif k == "end":
             if isinstance(v, str):
                 v = v.split(',')
@@ -144,13 +164,31 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
                     startquery["$and"].append(enddictvalue)
                 query["$and"].append(startquery)    
             elif isBracket==False:
-                v = str(startvalue[0])+','+str(9999999999)
-                v = v.split(',')
-                filters = generate_position_filter_end(self, k, v)
+                if isinstance(v, list) and isinstance(startvalue, list):
+                    startvalue=startvalue[0]
+                    v = str(v[0])
+                    stage2v = str(startvalue)+','+v
+                elif isinstance(v, list):
+                    v = str(v[0])
+                    stage2v = str(startvalue)+','+v
+                elif isinstance(startvalue, list):
+                    startvalue=startvalue[0]
+                    stage2v = str(startvalue)+','+str(v)
+                else:
+                    stage2v = str(startvalue)+','+str(v)
+                stage2v = stage2v.split(',')
+                filters = generate_position_filter_end(self, k, stage2v)
                 for filter in filters:
                     enddictvalue=apply_alphanumeric_filter(self, {}, filter, collection, dataset, True)
-                    startquery["$and"].append(enddictvalue)
-                query["$and"].append(startquery)    
+                    endquery["$and"].append(enddictvalue)
+                subqueryor["$or"].append(endquery)
+                stage2v = str(int(v)+1)+','+str(9999999999)
+                stage2v =stage2v.split(',')
+                filters = generate_position_filter_end(self, k, stage2v)
+                for filter in filters:
+                    enddictvalue=apply_alphanumeric_filter(self, {}, filter, collection, dataset, True)
+                    length_query["$and"].append(enddictvalue)    
+                length_query["$and"].append({'length': {'$gte': int(v)-int(startvalue)}}) 
 
 
         elif k == "datasets":
@@ -173,7 +211,7 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
                 raise web.HTTPNotFound    
         elif k == "mateName" or k == 'referenceName':
             try:
-                subqueryor["$or"].append(apply_alphanumeric_filter(self, {}, AlphanumericFilter(
+                referencedict["$or"].append(apply_alphanumeric_filter(self, {}, AlphanumericFilter(
                     id=VARIANTS_PROPERTY_MAP[k],
                     value=v
                 ), collection, dataset, True))
@@ -199,11 +237,18 @@ def apply_request_parameters(self, query: Dict[str, List[dict]], qparams: Reques
                 v_dict['id']=id
                 qparams.query.filters.append(v_dict)        
             return query, True
+    if length_query["$and"]!=[]:
+        subqueryor["$or"].append(length_query) 
     try:
         if subqueryor["$or"] != []:
             subquery["$and"].append(subqueryor)
     except Exception:# pragma: no cover
         pass
+    try:
+        if referencedict["$or"] != []:
+            query["$and"].append(referencedict)
+    except Exception:# pragma: no cover
+        pass
     if subquery["$and"] != []:
         query["$and"].append(subquery)
     elif startquery["$and"] != []: