Unify trigger

.github/workflows/buildContainer.yaml

@@ -118,7 +118,7 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.plugin }}/${{ matrix.plugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+          cp Images/${{ matrix.plugin }}/start_${{ matrix.plugin }}.sh Images/test_plugin.sh Images/${{ matrix.plugin }}/requirements.txt Images/trigger.py .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4
@@ -193,7 +193,7 @@ jobs:
         run: |
           VERSION=$(head -n 1 Images/${{ matrix.multiplugin }}/${{ matrix.multiplugin }}.dockerfile | cut -d ":" -f 2- | tr -d " ")
           echo "IMAGE_TAG=$VERSION" >> $GITHUB_ENV
-          cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger_noconda.py Images/trigger.py .
+          cp Images/${{ matrix.multiplugin }}/start_${{ matrix.multiplugin }}.sh Images/test_plugin.sh Images/${{ matrix.multiplugin }}/requirements.txt Images/trigger.py .
 
       - name: Download certificates from job build_main
         uses: actions/download-artifact@v4

Images/qp-deblur/qp-deblur.dockerfile

@@ -105,7 +105,7 @@ RUN sed -i "s|/opt/conda/envs/deblur/share/fragment-insertion/sepp/.sepp/bundled
 
 # install tornado based trigger layer in base environment
 RUN pip install -U --no-cache-dir tornado pip-system-certs
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 # ^^ 848 MB
 
 WORKDIR /

Images/qp-target-gene/qp-target-gene.dockerfile

@@ -112,7 +112,7 @@ RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 
 RUN pip3 install tornado
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 # copy sortmerna binaries
 COPY --from=builder /sortmerna-2.0/sortmerna /usr/local/bin/sortmerna

Images/qtp-biom/qtp-biom.dockerfile

@@ -135,7 +135,7 @@ RUN ln -s /usr/local/lib/python3.8/site-packages/scikit_learn.libs/libgomp-a34b3
 
 # install tornado based trigger layer in base environment
 #RUN pip install -U --no-cache-dir tornado
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 WORKDIR /
 

Images/qtp-diversity/qtp-diversity.dockerfile

@@ -119,7 +119,7 @@ RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 
 # install tornado based trigger layer in base environment
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 #COPY --from=builder /opt/conda/envs/dependencies/bin/* /opt/conda/bin/
 #COPY --from=builder /opt/conda/envs/dependencies/sbin/* /opt/conda/sbin/
@@ -156,7 +156,6 @@ RUN for f in `echo "libssu.so libhdf5_cpp.so.103 liblapacke.so.3 libcblas.so.3 l
 # fix an pandas deprecation issue, i.e. patch q2templates code
 RUN sed -i "s/'display.max_colwidth', -1/'display.max_colwidth', None/" /usr/local/lib/python3.8/site-packages/q2templates/util.py
 
-# COPY trigger_noconda.py /trigger.py
 ENV LC_ALL=C.UTF-8
 ENV LANG=C.UTF-8
 

Images/qtp-job-output-folder/qtp-job-output-folder.dockerfile

@@ -74,7 +74,7 @@ COPY --from=builder /wheels /wheels
 RUN pip install --no-cache-dir /wheels/* \
 	&& rm -rf rm -rf `find /usr/local/lib/python3.6/site-packages -type d -name "tests" | grep -v numpy`
 
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 COPY start_qtp-job-output-folder.sh .
 RUN chmod 755 start_qtp-job-output-folder.sh

Images/qtp-sequencing/qtp-sequencing.dockerfile

@@ -78,7 +78,7 @@ COPY --from=builder /opt/conda/envs/qtp-sequencing/lib/libdeflate.so.0 /lib/x86_
 # "install" pigz
 COPY --from=builder /opt/conda/envs/qtp-sequencing/bin/pigz /usr/local/bin/
 
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 # link to quast program
 RUN ln -s /usr/local/bin/quast.py /usr/local/bin/quast

Images/qtp-visualization/qtp-visualization.dockerfile

@@ -102,7 +102,7 @@ RUN chmod 755 start_qtp-visualization.sh
 RUN mkdir -p /unshared_plugins
 ENV QIITA_PLUGINS_DIR=/unshared_plugins/
 
-COPY trigger_noconda.py /trigger.py
+COPY trigger.py /trigger.py
 
 ##  Export cert and config filepaths
 COPY qiita_server_certificates/qiita_server_certificates.pem /qiita_server_certificates/qiita_server_certificates.pem

Images/trigger.py

@@ -5,11 +5,13 @@
 from glob import glob
 import sys
 import traceback
+import os
 import asyncio
 
 conda_env_name = None
 plugin_start_script = None
 plugin_src_dir = None
+use_conda = False
 
 class RunCommandHandler(tornado.web.RequestHandler):
     async def post(self):
@@ -28,7 +30,12 @@ async def post(self):
                 return
 
             # Systembefehl ausfuehren
-            cmd = 'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/%s; %s/scripts/%s %s %s %s' % (conda_env_name, plugin_src_dir, plugin_start_script, qiita_worker_url, job_id, output_dir)
+            cmd = ""
+            if use_conda:
+                cmd = 'source /opt/conda/etc/profile.d/conda.sh; conda activate /opt/conda/envs/%s; %s/scripts/%s %s %s %s' % (conda_env_name, plugin_src_dir, plugin_start_script, qiita_worker_url, job_id, output_dir)
+            else:
+                cmd = '%s %s %s %s' % (plugin_start_script, qiita_worker_url, job_id, output_dir)
+
             # Asynchronen Subprozess starten
             proc = await asyncio.create_subprocess_shell(
                 cmd,
@@ -52,11 +59,8 @@ async def post(self):
 
         except Exception as e:
             self.set_status(500)
-            # a hack to learn which docker service I am in
-            plugin_name = "unknown"
-            for f in glob('/start_*.sh'):
-                plugin_name = f.split('_')[-1].replace('.sh', '')
-                break
+            # I assume each image has set this env variable correctly!
+            plugin_name = os.environ["PLUGIN"]
             print("Error in service '%s': %s" % (plugin_name, str(e)), file=sys.stderr)
             traceback.print_exc()
             self.write({"error": str(e)})
@@ -78,9 +82,16 @@ def make_app():
     ])
 
 if __name__ == "__main__":
-    conda_env_name = sys.argv[1]
-    plugin_start_script = sys.argv[2]
-    plugin_src_dir = sys.argv[3]
+    if len(sys.argv) == 1:
+        use_conda = False
+        plugin_start_script = sys.argv[1]
+    elif len(sys.argv) == 3:
+        use_conda = True
+        conda_env_name = sys.argv[1]
+        plugin_start_script = sys.argv[2]
+        plugin_src_dir = sys.argv[3]
+    else:
+        raise ValueError("incorrect number of arguments provided for trigger.py!")
 
     app = make_app()
     app.listen(5000)  # Server auf Port 5000 starten

Makefile

@@ -31,7 +31,7 @@ $(DIR_REFERENCES)/qiita_server_certificates: Images/plugin_collector/stefan_csr.
 	# === end: create own certificates ===
 
 # a general target, executed for each plugin
-plugin: Images/trigger.py Images/trigger_noconda.py $(DIR_REFERENCES)/qiita_server_certificates Images/test_plugin.sh 
+plugin: Images/trigger.py $(DIR_REFERENCES)/qiita_server_certificates Images/test_plugin.sh
 	cp -r $^ $(tmpdir)/
 
 .built_image_qtp-biom: Images/qtp-biom/qtp-biom.dockerfile Images/qtp-biom/start_qtp-biom.sh src/qiita-files/ src/qtp-biom/ Images/qtp-biom/requirements.txt