Merge branch 'main' into status-waiting-time

* main:
  Hotfix for check on frequency provider
  Tests run_until must be guard-claused with cleanup routine (#616)
  Fix check if stderr is empty (#613)
  Bump uvicorn[standard] from 0.24.0.post1 to 0.25.0 (#612)
  Fxing the network provider stderror
  Branch and filename are now always not null (#602)
  Adds a more elaborate depends_on test
  Support reading notes from services (#590)
  docker build command in tests now checks reason for docker build failure. If it is a permission issue with the cache, it will continue the rest of the workflow (#576)
  Use depends_on for container startup order (refactored) (#593)
  Bump psycopg[binary] from 3.1.15 to 3.1.16 (#610)
  Added powercap info to hardware_info (#609)
  Changed wording for network infrastructure box (#608)
  Added SIGQUIT to nginx and initi to gunicorn, as we are using bash script in entrypoint (#605)
  Fix frontend flow menu to wrap automatically (#584)
  Bump psutil from 5.9.6 to 5.9.7 (#603)
  Disable Docker CLI hints (#555)
  Create codeql.yml

Author: ArneTR

.github/actions/gmt-pytest/action.yml

@@ -60,9 +60,21 @@ runs:
         password: ${{ inputs.github-token }}
 
     - name: Build docker-compose
+      id: build-docker-compose
       shell: bash
       working-directory: ${{ inputs.gmt-directory }}/docker
-      run: docker buildx bake --file test-compose.yml --file docker-compose-cache.json
+      run: |
+        { DOCKER_OUTPUT=$(docker buildx bake --file test-compose.yml --file docker-compose-cache.json 2>&1); DOCKER_EXIT_CODE=$?; } || true
+        if [ "$DOCKER_EXIT_CODE" -ne 0 ]; then
+          echo "Docker build failed with exit code $DOCKER_EXIT_CODE"
+          echo "buildx output:"
+          echo $DOCKER_OUTPUT
+          if echo "$DOCKER_OUTPUT" | grep -q "403 Forbidden"; then
+            echo "Docker build failed due to permissions issue. Continuing..."
+          else
+              exit 1
+          fi
+        fi
 
     - name: Start Test container
       shell: bash

.github/workflows/codeql.yml

@@ -0,0 +1,79 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'javascript-typescript', 'python' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@v3
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        queries: security-extended
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@v3
+      with:
+        category: "/language:${{matrix.language}}"

.gitignore

@@ -19,3 +19,4 @@ tools/sgx_enable
 venv/
 lib/hardware_info_root.py
 tools/cluster/cleanup.sh
+node_modules/

api/api_helpers.py

@@ -108,17 +108,15 @@ def get_run_info(run_id):
     return DB().fetch_one(query, params=params, row_factory=psycopg_rows_dict_row)
 
 
-def get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_date=None, end_date=None, detail_name=None, limit_365=False, sorting='run'):
+def get_timeline_query(uri, filename, machine_id, branch, metrics, phase, start_date=None, end_date=None, detail_name=None, limit_365=False, sorting='run'):
 
     if filename is None or filename.strip() == '':
         filename =  'usage_scenario.yml'
 
-    params = [uri, filename, machine_id, f"%{phase}"]
+    if branch is None or branch.strip() != '':
+        branch = 'main'
 
-    branch_condition = 'AND r.branch IS NULL'
-    if branch is not None and branch.strip() != '':
-        branch_condition = 'AND r.branch = %s'
-        params.append(branch)
+    params = [uri, filename, branch, machine_id, f"%{phase}"]
 
     metrics_condition = ''
     if metrics is None or metrics.strip() == '' or metrics.strip() == 'key':
@@ -161,10 +159,10 @@ def get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_da
             WHERE
                 r.uri = %s
                 AND r.filename = %s
+                AND r.branch = %s
                 AND r.end_measurement IS NOT NULL
                 AND r.machine_id = %s
                 AND p.phase LIKE %s
-                {branch_condition}
                 {metrics_condition}
                 {start_date_condition}
                 {end_date_condition}
@@ -176,13 +174,14 @@ def get_timeline_query(uri,filename,machine_id, branch, metrics, phase, start_da
                 p.phase ASC, {sorting_condition}
 
             """
+
     return (query, params)
 
 def determine_comparison_case(ids):
 
     query = '''
             WITH uniques as (
-                SELECT uri, filename, machine_id, commit_hash, COALESCE(branch, 'main / master') as branch FROM runs
+                SELECT uri, filename, machine_id, commit_hash, branch FROM runs
                 WHERE id = ANY(%s::uuid[])
                 GROUP BY uri, filename, machine_id, commit_hash, branch
             )
@@ -282,7 +281,7 @@ def get_phase_stats(ids):
     query = """
             SELECT
                 a.phase, a.metric, a.detail_name, a.value, a.type, a.max_value, a.min_value, a.unit,
-                b.uri, c.description, b.filename, b.commit_hash, COALESCE(b.branch, 'main / master') as branch
+                b.uri, c.description, b.filename, b.commit_hash, b.branch
             FROM phase_stats as a
             LEFT JOIN runs as b on b.id = a.run_id
             LEFT JOIN machines as c on c.id = b.machine_id

api/main.py

@@ -227,7 +227,7 @@ async def get_repositories(uri: str | None = None, branch: str | None = None, ma
 async def get_runs(uri: str | None = None, branch: str | None = None, machine_id: int | None = None, machine: str | None = None, filename: str | None = None, limit: int | None = None):
 
     query = """
-            SELECT r.id, r.name, r.uri, COALESCE(r.branch, 'main / master'), r.created_at, r.invalid_run, r.filename, m.description, r.commit_hash, r.end_measurement
+            SELECT r.id, r.name, r.uri, r.branch, r.created_at, r.invalid_run, r.filename, m.description, r.commit_hash, r.end_measurement
             FROM runs as r
             LEFT JOIN machines as m on r.machine_id = m.id
             WHERE 1=1
@@ -303,7 +303,7 @@ async def compare_in_repo(ids: str):
         machine = machines[run_info['machine_id']]
         uri = run_info['uri']
         usage_scenario = run_info['usage_scenario']['name']
-        branch = run_info['branch'] if run_info['branch'] is not None else 'main / master'
+        branch = run_info['branch']
         commit = run_info['commit_hash']
         filename = run_info['filename']
 
@@ -519,8 +519,8 @@ async def get_timeline_projects():
                 FROM runs as r
                 WHERE
                     p.url = r.uri
-                    AND COALESCE(p.branch, 'main / master') = COALESCE(r.branch, 'main / master')
-                    AND COALESCE(p.filename, 'usage_scenario.yml') = COALESCE(r.filename, 'usage_scenario.yml')
+                    AND p.branch = r.branch
+                    AND p.filename = r.filename
                     AND p.machine_id = r.machine_id
                 ORDER BY r.created_at DESC
                 LIMIT 1
@@ -1017,15 +1017,15 @@ async def software_add(software: Software):
     if software.email is None or software.email.strip() == '':
         raise RequestValidationError('E-mail is empty')
 
-    if not DB().fetch_one('SELECT id FROM machines WHERE id=%s AND available=TRUE', params=(software.machine_id,)):
-        raise RequestValidationError('Machine does not exist')
+    if software.branch is None or software.branch.strip() == '':
+        software.branch = 'main'
 
+    if software.filename is None or software.filename.strip() == '':
+        software.filename = 'usage_scenario.yml'
 
-    if software.branch.strip() == '':
-        software.branch = None
+    if not DB().fetch_one('SELECT id FROM machines WHERE id=%s AND available=TRUE', params=(software.machine_id,)):
+        raise RequestValidationError('Machine does not exist')
 
-    if software.filename.strip() == '':
-        software.filename = 'usage_scenario.yml'
 
     if software.schedule_mode not in ['one-off', 'time', 'commit', 'variance']:
         raise RequestValidationError(f"Please select a valid measurement interval. ({software.schedule_mode}) is unknown.")

config.yml.example

@@ -44,6 +44,8 @@ measurement:
   idle-time-end: 5
   flow-process-runtime: 3800
   phase-transition-time: 1
+  boot:
+    wait_time_dependencies: 20
   metric-providers:
 
   # Please select the needed providers according to the working ones on your system

docker/compose.yml.example

@@ -32,6 +32,7 @@ services:
     # should not run on the measurement node anyway
     image: nginx
     container_name: green-coding-nginx-container
+    stop_signal: SIGQUIT
     depends_on:
       - green-coding-postgres
     ports:
@@ -49,6 +50,7 @@ services:
       context: .
       dockerfile: Dockerfile-gunicorn
     container_name: green-coding-gunicorn-container
+    init: true
     depends_on:
       - green-coding-postgres
     restart: always

docker/requirements.txt

@@ -1,7 +1,7 @@
 gunicorn==21.2.0
-psycopg[binary]==3.1.15
+psycopg[binary]==3.1.16
 fastapi==0.105.0
-uvicorn[standard]==0.24.0.post1
+uvicorn[standard]==0.25.0
 pandas==2.1.4
 PyYAML==6.0.1
 anybadge==1.14.0

docker/structure.sql

@@ -24,8 +24,8 @@ CREATE TABLE jobs (
     name text,
     email text,
     url text,
-    branch text,
-    filename text,
+    branch text NOT NULL,
+    filename text NOT NULL,
     categories int[],
     machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE,
     created_at timestamp with time zone DEFAULT now(),
@@ -41,13 +41,13 @@ CREATE TABLE runs (
     job_id integer REFERENCES jobs(id) ON DELETE SET NULL ON UPDATE CASCADE UNIQUE,
     name text,
     uri text,
-    branch text,
+    branch text NOT NULL,
     commit_hash text,
     commit_timestamp timestamp with time zone,
     email text,
     categories int[],
     usage_scenario json,
-    filename text,
+    filename text NOT NULL,
     machine_specs jsonb,
     runner_arguments json,
     machine_id int REFERENCES machines(id) ON DELETE SET NULL ON UPDATE CASCADE,
@@ -198,8 +198,8 @@ CREATE TABLE timeline_projects (
     name text,
     url text,
     categories integer[],
-    branch text DEFAULT 'NULL'::text,
-    filename text,
+    branch text NOT NULL,
+    filename text NOT NULL,
     machine_id integer REFERENCES machines(id) ON DELETE RESTRICT ON UPDATE CASCADE NOT NULL,
     schedule_mode text NOT NULL,
     last_scheduled timestamp with time zone,

frontend/compare.html

@@ -187,7 +187,7 @@ <h3>Single Phase Data</h3>
                 Runtime can contain multiple flows.
                 By default all runtime flows are aggregated. Please select a separate flow if needed.
             </div>
-            <div id="runtime-sub-phases" class="ui top attached tabular menu">
+            <div id="runtime-sub-phases" class="ui top attached tabular wrapping menu">
                 <a class="active item" data-tab="[[RUNTIME]]">All Flows</a>
                 <a class="item runtime-step" data-tab="" style="display:none;"></a> <!-- empty element for copying. Usage phases will be inserted before -->
             </div>