action.yml

name: redline eval
description: Run redline as a local-first prompt regression gate in GitHub Actions.
author: Gowtham Sarveswaran

branding:
  icon: activity
  color: red

inputs:
  config:
    description: Path to redline.json.
    default: redline.json
  suite:
    description: Path to the committed redline suite or prompt manifest.
    default: redline-suite.json
  prompt-path:
    description: Optional prompt template file to evaluate.
    default: ""
  fail-on:
    description: Comma-separated statuses that fail the action.
    default: regression,missing
  working-directory:
    description: Repository directory where redline should run.
    default: "."
  extra-args:
    description: Space-separated extra arguments appended to redline eval. Values containing spaces are not supported; use redline.json for complex values.
    default: ""
  record-history:
    description: Append eval results to .redline/history.jsonl and GitHub summary.
    default: "true"
  history-label:
    description: Label for the history entry. Defaults to GITHUB_SHA when empty.
    default: ""
  render-dashboard:
    description: Render .redline/dashboard.html after eval.
    default: "true"
  benchmark-max-seconds:
    description: Optional maximum worst-case eval budget in seconds. Empty means report only.
    default: ""

runs:
  using: composite
  steps:
    - name: Install redline
      shell: bash
      run: python -m pip install "${{ github.action_path }}"

    - name: Run redline
      shell: bash
      env:
        REDLINE_ACTION_CONFIG: ${{ inputs.config }}
        REDLINE_ACTION_SUITE: ${{ inputs.suite }}
        REDLINE_ACTION_PROMPT_PATH: ${{ inputs.prompt-path }}
        REDLINE_ACTION_FAIL_ON: ${{ inputs.fail-on }}
        REDLINE_ACTION_WORKING_DIRECTORY: ${{ inputs.working-directory }}
        REDLINE_ACTION_EXTRA_ARGS: ${{ inputs.extra-args }}
        REDLINE_ACTION_RECORD_HISTORY: ${{ inputs.record-history }}
        REDLINE_ACTION_HISTORY_LABEL: ${{ inputs.history-label }}
        REDLINE_ACTION_RENDER_DASHBOARD: ${{ inputs.render-dashboard }}
        REDLINE_ACTION_BENCHMARK_MAX_SECONDS: ${{ inputs.benchmark-max-seconds }}
      run: |
        set -euo pipefail
        cd "$REDLINE_ACTION_WORKING_DIRECTORY"
        mkdir -p .redline/reports

        config_path="$REDLINE_ACTION_CONFIG"
        suite_path="$REDLINE_ACTION_SUITE"
        if [ ! -f "$suite_path" ]; then
          python -m redline doctor --config "$config_path" || true
          echo "::error::redline suite or prompt manifest not found: $suite_path"
          echo "::error::Generate and commit one with: redline suite path/to/baseline.jsonl --out $suite_path"
          exit 1
        fi
        suite_schema="$(python -c 'import json, sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("schema", ""))' "$suite_path")"

        python -m redline doctor --config "$config_path" --strict
        if [ "$suite_schema" = "redline-prompt-manifest-v1" ]; then
          if [ -n "$REDLINE_ACTION_PROMPT_PATH" ]; then
            echo "::error::prompt-path is not supported when suite points to a redline prompt manifest"
            exit 1
          fi
          manifest_root="$(python -c 'import json, sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("root", "prompts"))' "$suite_path")"
          manifest_suite_dir="$(python -c 'import json, sys; print(json.load(open(sys.argv[1], encoding="utf-8")).get("suite_dir", "suites"))' "$suite_path")"
          python -m redline prompts "$manifest_root" \
            --suite-dir "$manifest_suite_dir" \
            --out "$suite_path" \
            --check \
            --check-suites
        else
          python -m redline validate "$suite_path" --config "$config_path" --strict
        fi
        benchmark_args=(
          "$suite_path"
          --config "$config_path"
          --measure-local
          --github-summary
          --out-json .redline/reports/benchmark.json
          --out-md .redline/reports/benchmark.md
        )
        if [ -n "$REDLINE_ACTION_BENCHMARK_MAX_SECONDS" ]; then
          benchmark_args+=(--max-seconds "$REDLINE_ACTION_BENCHMARK_MAX_SECONDS")
        fi
        python -m redline budget "${benchmark_args[@]}"

        eval_args=(
          "$suite_path"
          --config "$config_path"
          --compact
          --github-annotations
          --fail-on "$REDLINE_ACTION_FAIL_ON"
          --out-json .redline/reports/eval.json
          --out-md .redline/reports/eval.md
          --out-comment .redline/reports/eval-comment.md
          --out-html .redline/reports/eval.html
          --out-junit .redline/reports/eval.xml
          --out-slack .redline/reports/eval.slack.json
        )
        if [ -n "$REDLINE_ACTION_PROMPT_PATH" ]; then
          eval_args+=(--prompt "$REDLINE_ACTION_PROMPT_PATH")
        fi
        if [ -n "$REDLINE_ACTION_EXTRA_ARGS" ]; then
          read -r -a extra_args <<< "$REDLINE_ACTION_EXTRA_ARGS"
          eval_args+=("${extra_args[@]}")
        fi

        set +e
        python -m redline eval "${eval_args[@]}"
        redline_status=$?
        set -e

        if [ -n "${GITHUB_STEP_SUMMARY:-}" ] && [ -f .redline/reports/eval-comment.md ]; then
          cat .redline/reports/eval-comment.md >> "$GITHUB_STEP_SUMMARY"
        fi

        if [ "$REDLINE_ACTION_RECORD_HISTORY" = "true" ] && [ -f .redline/reports/eval.json ]; then
          history_label="$REDLINE_ACTION_HISTORY_LABEL"
          if [ -z "$history_label" ]; then
            history_label="${GITHUB_SHA:-redline-action}"
          fi
          python -m redline history \
            .redline/reports/eval.json \
            --label "$history_label" \
            --out .redline/history.jsonl \
            --out-md .redline/history.md \
            --github-summary \
            --fail-on none
        fi

        python -m redline audit \
          --config "$config_path" \
          --verify \
          --out-checkpoint .redline/audit-checkpoint.json

        if [ -n "${GITHUB_STEP_SUMMARY:-}" ] && [ -f .redline/audit-checkpoint.json ]; then
          {
            echo ""
            echo "### Audit evidence"
            echo ""
            echo '- Checkpoint: `.redline/audit-checkpoint.json`'
            echo '- Verify locally: `redline audit --verify --checkpoint .redline/audit-checkpoint.json`'
          } >> "$GITHUB_STEP_SUMMARY"
        fi

        if [ "$REDLINE_ACTION_RENDER_DASHBOARD" = "true" ]; then
          python -m redline dashboard --out .redline/dashboard.html
        fi

        exit "$redline_status"