Repository navigation
Daily Trading Bot #291
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Daily Trading Bot | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: false | |
| on: | |
| schedule: | |
| # Runs at 21:30 UTC (3:00 AM IST next day / 5:30 AM SGT next day) Monday-Friday | |
| # to ensure the US stock market is closed (4:00 PM EST/EDT) and final daily close prices are settled. | |
| - cron: '30 21 * * 1-5' | |
| workflow_dispatch: | |
| inputs: | |
| run_in_lightning_studio: | |
| description: "Run the full daily job inside Lightning Studio with local trained-model inference" | |
| required: false | |
| default: false | |
| type: boolean | |
| disable_ai_trading: | |
| description: "Run only the core bot and skip AI trading" | |
| required: false | |
| default: false | |
| type: boolean | |
| disable_core_trading: | |
| description: "Run only the AI bot and skip Core trading" | |
| required: false | |
| default: false | |
| type: boolean | |
| reset_ai_positions: | |
| description: "Clear only AI positions before the manual run so the trained model must generate fresh trades" | |
| required: false | |
| default: false | |
| type: boolean | |
| core_cache_restore_key: | |
| description: "Optional exact Actions cache key for restoring core state (example: Linux-trading-bot-24061212677)" | |
| required: false | |
| default: "" | |
| type: string | |
| reset_core_database: | |
| description: "Reset the Core database before running" | |
| required: false | |
| default: false | |
| type: boolean | |
| jobs: | |
| run-trading-bot: | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 240 | |
| permissions: | |
| contents: read | |
| env: | |
| TZ: Asia/Singapore | |
| MODAL_INFERENCE_URL: ${{ secrets.MODAL_INFERENCE_URL }} | |
| MODAL_TRAINED_MODEL_URL: ${{ secrets.MODAL_TRAINED_MODEL_URL }} | |
| MODAL_API_KEY: ${{ secrets.MODAL_API_KEY || secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_INFERENCE_URL: ${{ secrets.TRAINED_MODEL_INFERENCE_URL }} | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "300" | |
| TRAINED_MODEL_READY_TIMEOUT_SECONDS: "1200" | |
| TRAINED_MODEL_READY_POLL_SECONDS: "15" | |
| TRAINED_MODEL_MAX_RETRIES: "0" | |
| TRAINED_MODEL_BACKOFF_SECONDS: "5" | |
| TRAINED_MODEL_BATCH_SIZE: "16" | |
| LIGHTNING_USERNAME: ${{ secrets.LIGHTNING_USERNAME }} | |
| LIGHTNING_API_KEY: ${{ secrets.LIGHTNING_API_KEY }} | |
| LIGHTNING_USER_ID: ${{ secrets.LIGHTNING_USER_ID }} | |
| LIGHTNING_CLOUD_PROJECT_ID: "01kkqt8mzptzp8svph6p4nt915" | |
| GITHUB_TOKEN: ${{ github.token }} | |
| TRAINED_MODEL_BASE_MODEL: "Qwen/Qwen2.5-7B-Instruct" | |
| TRAINED_MODEL_NAME: "quant-trained-trading-model" | |
| TRAINED_MODEL_CPU_THREADS: "4" | |
| LIGHTNING_INFERENCE_COMPUTE_NAME: "cpu-4" | |
| LIGHTNING_INFERENCE_DISK_GB: "40" | |
| LIGHTNING_INFERENCE_PORT: "8000" | |
| TRAINED_MODEL_ADAPTER_ARCHIVE_URL: ${{ secrets.TRAINED_MODEL_ADAPTER_ARCHIVE_URL }} | |
| TRAINED_MODEL_ADAPTER_ARCHIVE_TOKEN: ${{ secrets.TRAINED_MODEL_ADAPTER_ARCHIVE_TOKEN }} | |
| steps: | |
| - name: Checkout repository | |
| uses: actions/checkout@v4 | |
| - name: Enforce schedule day guard (Mon-Fri only) | |
| if: ${{ github.event_name == 'schedule' }} | |
| run: | | |
| dow=$(date -u +%u) | |
| # ISO day-of-week: 1=Mon ... 7=Sun. Allow Mon(1) through Fri(5) only. | |
| if [ "${dow}" -lt 1 ] || [ "${dow}" -gt 5 ]; then | |
| echo "Scheduled run blocked: UTC weekday ${dow} is outside Mon-Fri window." >&2 | |
| exit 1 | |
| fi | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.10' | |
| cache: 'pip' | |
| - name: Install dependencies | |
| id: install_base_dependencies | |
| continue-on-error: true | |
| timeout-minutes: 20 | |
| run: | | |
| python -m pip install --upgrade pip | |
| pip install -r requirements.txt | |
| - name: Send Base Dependency Failure Reports | |
| if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && steps.install_base_dependencies.outcome == 'failure' }} | |
| continue-on-error: true | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} | |
| run: | | |
| python send_failure_report.py \ | |
| --strategy-tag Core \ | |
| --source "Install dependencies" \ | |
| --message "Base Python dependencies failed to install before the core daily job could start. Run: ${RUN_URL}" | |
| python send_failure_report.py \ | |
| --strategy-tag AI \ | |
| --source "Install dependencies" \ | |
| --message "Base Python dependencies failed to install before the AI daily job could start. Run: ${RUN_URL}" \ | |
| --model-used "quant-trained-trading-model" | |
| - name: Restore Core Cache (Exact Key) | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && github.event_name == 'workflow_dispatch' && inputs.core_cache_restore_key != '' }} | |
| id: restore_core_cache_exact | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| data/ | |
| models/ | |
| model_registry.json | |
| results/ | |
| key: ${{ inputs.core_cache_restore_key }} | |
| - name: Verify Exact Core Cache Hit | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && github.event_name == 'workflow_dispatch' && inputs.core_cache_restore_key != '' }} | |
| run: | | |
| if [ "${{ steps.restore_core_cache_exact.outputs.cache-hit }}" != "true" ]; then | |
| echo "Exact cache key not found: ${{ inputs.core_cache_restore_key }}" >&2 | |
| exit 1 | |
| fi | |
| - name: Restore Data and Models Cache (Latest Prefix) | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && !(github.event_name == 'workflow_dispatch' && inputs.core_cache_restore_key != '') }} | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| data/ | |
| models/ | |
| model_registry.json | |
| results/ | |
| key: ${{ runner.os }}-trading-bot-${{ github.run_id }} | |
| restore-keys: | | |
| ${{ runner.os }}-trading-bot- | |
| - name: Run Trading Bot | |
| id: run_core_bot | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_core_trading) }} | |
| continue-on-error: true | |
| timeout-minutes: 45 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_CORE_API_KEY: ${{ secrets.ALPACA_CORE_API_KEY }} | |
| ALPACA_CORE_SECRET_KEY: ${{ secrets.ALPACA_CORE_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_AI_TRADING: "1" | |
| RESET_CORE_DATABASE: ${{ github.event.inputs.reset_core_database || 'false' }} | |
| run: | | |
| set +e | |
| python main.py daily_job | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Run Trading Bot (Core Retry) | |
| id: run_core_bot_retry | |
| if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_core_trading) && steps.run_core_bot.outputs.rc != '0' }} | |
| continue-on-error: true | |
| timeout-minutes: 45 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_CORE_API_KEY: ${{ secrets.ALPACA_CORE_API_KEY }} | |
| ALPACA_CORE_SECRET_KEY: ${{ secrets.ALPACA_CORE_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_AI_TRADING: "1" | |
| RESET_CORE_DATABASE: ${{ github.event.inputs.reset_core_database || 'false' }} | |
| run: | | |
| set +e | |
| python main.py daily_job | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Send Core Failure Report | |
| if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_core_trading) && steps.run_core_bot.outputs.rc != '0' && steps.run_core_bot_retry.outputs.rc != '0' }} | |
| continue-on-error: true | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} | |
| run: | | |
| python send_failure_report.py \ | |
| --strategy-tag Core \ | |
| --source "Run Trading Bot" \ | |
| --message "Core daily job failed after retry and did not confirm its report email. Run: ${RUN_URL}" | |
| - name: Install Lightning dependencies | |
| id: install_lightning_deps | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && ((github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) || ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading)) }} | |
| continue-on-error: ${{ !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) }} | |
| timeout-minutes: 20 | |
| run: | | |
| python -m pip install "lightning[app]==2.3.2" lightning-cloud==0.5.70 | |
| - name: Validate SMTP configuration for AI reports | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| run: | | |
| missing="" | |
| for name in SMTP_SERVER SMTP_PORT SENDER_EMAIL SENDER_PASSWORD RECIPIENT_EMAIL; do | |
| if [ -z "${!name}" ]; then | |
| missing="${missing} ${name}" | |
| fi | |
| done | |
| if [ -n "${missing}" ]; then | |
| echo "Missing required SMTP env vars for AI report:${missing}" >&2 | |
| exit 1 | |
| fi | |
| - name: Plan AI runtime | |
| id: plan_ai_runtime | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) }} | |
| run: | | |
| mkdir -p results | |
| configured_inference_url="${MODAL_TRAINED_MODEL_URL}" | |
| if [ -z "${configured_inference_url}" ]; then configured_inference_url="${MODAL_INFERENCE_URL}"; fi | |
| if [ -z "${configured_inference_url}" ]; then configured_inference_url="${TRAINED_MODEL_INFERENCE_URL}"; fi | |
| if [ -n "${configured_inference_url}" ]; then | |
| runtime_mode="modal_full" | |
| reason="modal_primary_configured" | |
| # Enable Lightning as reactive fallback for scheduled runs | |
| enable_lightning_fallback="true" | |
| lightning_compute_name="cpu-4" | |
| lightning_disk_gb="40" | |
| else | |
| runtime_mode="distilled_local" | |
| reason="modal_inference_url_missing" | |
| enable_lightning_fallback="false" | |
| lightning_compute_name="" | |
| lightning_disk_gb="0" | |
| fi | |
| { | |
| echo "runtime_mode=${runtime_mode}" | |
| echo "has_inference_url=$([ -n "${configured_inference_url}" ] && echo true || echo false)" | |
| echo "resolved_inference_url=${configured_inference_url}" | |
| echo "reason=${reason}" | |
| echo "enable_lightning_fallback=${enable_lightning_fallback}" | |
| echo "lightning_compute_name=${lightning_compute_name}" | |
| echo "lightning_disk_gb=${lightning_disk_gb}" | |
| } >> "$GITHUB_OUTPUT" | |
| echo "AI runtime decided: ${runtime_mode} (${reason})" | |
| - name: Warm Modal inference app | |
| id: warm_modal_inference | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && steps.plan_ai_runtime.outputs.has_inference_url == 'true' }} | |
| continue-on-error: true | |
| timeout-minutes: 45 | |
| env: | |
| AI_PRIMARY_BACKEND: "modal" | |
| MODAL_INFERENCE_URL: ${{ steps.plan_ai_runtime.outputs.resolved_inference_url }} | |
| MODAL_API_KEY: ${{ secrets.MODAL_API_KEY || secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_MAX_RETRIES: "2" | |
| TRAINED_MODEL_BACKOFF_SECONDS: "10" | |
| TRAINED_MODEL_READY_TIMEOUT_SECONDS: "1800" | |
| TRAINED_MODEL_READY_POLL_SECONDS: "20" | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "600" | |
| run: | | |
| python - <<'PYCODE' | |
| import os, time, requests | |
| url = (os.getenv("MODAL_INFERENCE_URL") or "").strip() | |
| if not url: | |
| raise SystemExit("MODAL_INFERENCE_URL is empty") | |
| key = (os.getenv("MODAL_API_KEY") or "").strip() | |
| headers = {"Accept": "application/json"} | |
| if key: | |
| headers["Authorization"] = f"Bearer {key}" | |
| roots = [url] | |
| if url.rstrip("/").endswith("/predict_trade_candidates"): | |
| roots.append(url[: -len("/predict_trade_candidates")]) | |
| else: | |
| roots.append(url.rstrip("/") + "/predict_trade_candidates") | |
| candidates = [] | |
| for item in roots: | |
| base = item.rstrip("/") | |
| if base.endswith("/predict_trade_candidates"): | |
| base = base[: -len("/predict_trade_candidates")] | |
| candidates.extend([base + "/health", base + "/warmup"]) | |
| seen, endpoints = set(), [] | |
| for e in candidates: | |
| if e not in seen: | |
| seen.add(e) | |
| endpoints.append(e) | |
| last_error = None | |
| for attempt in range(1, 8): | |
| for endpoint in endpoints: | |
| try: | |
| if endpoint.endswith("/health"): | |
| r = requests.get(endpoint, headers=headers, timeout=30) | |
| else: | |
| r = requests.post(endpoint, headers=headers, timeout=120) | |
| if r.status_code < 500 and r.status_code != 404: | |
| print("warm endpoint ok:", endpoint, "status=", r.status_code) | |
| raise SystemExit(0) | |
| print("warm endpoint retryable:", endpoint, "status=", r.status_code, "body=", (r.text or "")[:240]) | |
| last_error = RuntimeError(f"{endpoint} -> {r.status_code}") | |
| except Exception as exc: | |
| print("warm endpoint exception:", endpoint, str(exc)) | |
| last_error = exc | |
| time.sleep(min(20, 3 * attempt)) | |
| raise SystemExit(f"Modal endpoint warmup failed across candidate URLs: {last_error}") | |
| PYCODE | |
| - name: Verify Modal predict endpoint | |
| id: verify_modal_predict | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && steps.plan_ai_runtime.outputs.has_inference_url == 'true' }} | |
| continue-on-error: true | |
| timeout-minutes: 3 | |
| env: | |
| MODAL_INFERENCE_URL: ${{ steps.plan_ai_runtime.outputs.resolved_inference_url }} | |
| MODAL_API_KEY: ${{ secrets.MODAL_API_KEY || secrets.TRAINED_MODEL_API_KEY }} | |
| run: | | |
| python - <<'PYCODE' | |
| import json, os, requests | |
| url = (os.getenv('MODAL_INFERENCE_URL') or '').strip() | |
| if not url: | |
| raise SystemExit('MODAL_INFERENCE_URL is empty') | |
| payload = { | |
| 'task': 'trade_signal_classification', | |
| 'candidates': [{'symbol':'AAPL','as_of_date':'2026-05-13','return_1d':0.01,'return_5d':0.02,'return_10d':0.03,'volatility_20d':0.02,'dist_ma_20':0.01,'dist_ma_50':0.02,'rsi_14':55,'volume_ratio':1.1,'news_count_7d':1,'news_sentiment_7d':0.1}] | |
| } | |
| headers = {'Content-Type':'application/json','Accept':'application/json'} | |
| key = (os.getenv('MODAL_API_KEY') or '').strip() | |
| if key: | |
| headers['Authorization'] = f'Bearer {key}' | |
| candidate_urls = [url] | |
| if url.rstrip('/').endswith('/predict_trade_candidates'): | |
| candidate_urls.append(url[: -len('/predict_trade_candidates')]) | |
| else: | |
| candidate_urls.append(url.rstrip('/') + '/predict_trade_candidates') | |
| last_error = None | |
| max_attempts = 3 | |
| preflight_ok = False | |
| for test_url in candidate_urls: | |
| for attempt in range(1, max_attempts + 1): | |
| try: | |
| r = requests.post(test_url, json=payload, headers=headers, timeout=45) | |
| print('preflight url', test_url, 'attempt', attempt) | |
| print('status', r.status_code) | |
| print((r.text or '')[:500]) | |
| r.raise_for_status() | |
| data = r.json() | |
| if not isinstance(data, dict): | |
| raise SystemExit('Predict response was not JSON object') | |
| has_signals = isinstance(data.get('signals'), list) | |
| has_signal = data.get('signal') is not None | |
| if not (has_signals or has_signal): | |
| raise SystemExit('Predict response missing signal/signals') | |
| print('predict preflight ok') | |
| preflight_ok = True | |
| break | |
| except Exception as exc: | |
| last_error = exc | |
| print('preflight attempt failed:', exc) | |
| if attempt < max_attempts: | |
| import time | |
| time.sleep(10 * attempt) | |
| if preflight_ok: | |
| break | |
| if not preflight_ok: | |
| raise SystemExit(f'Predict preflight failed for all candidate URLs: {last_error}') | |
| PYCODE | |
| - name: Run AI Trading Bot on Modal | |
| id: run_ai_bot_modal | |
| if: ${{ steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && steps.plan_ai_runtime.outputs.has_inference_url == 'true' && steps.verify_modal_predict.outcome == 'success' }} | |
| continue-on-error: true | |
| timeout-minutes: 90 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| MODAL_INFERENCE_URL: ${{ steps.plan_ai_runtime.outputs.resolved_inference_url }} | |
| MODAL_API_KEY: ${{ secrets.MODAL_API_KEY || secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_AI_API_KEY: ${{ secrets.ALPACA_AI_API_KEY }} | |
| ALPACA_AI_SECRET_KEY: ${{ secrets.ALPACA_AI_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_CORE_TRADING: "1" | |
| AI_RUNTIME_MODE: "full" | |
| AI_PRIMARY_BACKEND: "modal" | |
| AI_ROUTER_REASON: ${{ steps.plan_ai_runtime.outputs.reason }} | |
| AI_PROMPT_CANDIDATES_LIMIT: "200" | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "600" | |
| TRAINED_MODEL_READY_TIMEOUT_SECONDS: "1800" | |
| TRAINED_MODEL_READY_POLL_SECONDS: "20" | |
| TRAINED_MODEL_MAX_RETRIES: "2" | |
| TRAINED_MODEL_BACKOFF_SECONDS: "10" | |
| TRAINED_MODEL_BATCH_SIZE: "16" | |
| run: | | |
| set +e | |
| python main.py daily_job | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Run AI Trading Bot on Modal (Retry) | |
| id: run_ai_bot_modal_retry | |
| if: ${{ always() && steps.install_base_dependencies.outcome == 'success' && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && steps.plan_ai_runtime.outputs.has_inference_url == 'true' && steps.verify_modal_predict.outcome == 'success' && steps.run_ai_bot_modal.outputs.rc != '0' }} | |
| continue-on-error: true | |
| timeout-minutes: 90 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| MODAL_INFERENCE_URL: ${{ steps.plan_ai_runtime.outputs.resolved_inference_url }} | |
| MODAL_API_KEY: ${{ secrets.MODAL_API_KEY || secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_AI_API_KEY: ${{ secrets.ALPACA_AI_API_KEY }} | |
| ALPACA_AI_SECRET_KEY: ${{ secrets.ALPACA_AI_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_CORE_TRADING: "1" | |
| AI_RUNTIME_MODE: "full" | |
| AI_PRIMARY_BACKEND: "modal" | |
| AI_ROUTER_REASON: "modal_retry_after_failure" | |
| AI_PROMPT_CANDIDATES_LIMIT: "200" | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "600" | |
| TRAINED_MODEL_READY_TIMEOUT_SECONDS: "1800" | |
| TRAINED_MODEL_READY_POLL_SECONDS: "20" | |
| TRAINED_MODEL_MAX_RETRIES: "2" | |
| TRAINED_MODEL_BACKOFF_SECONDS: "10" | |
| TRAINED_MODEL_BATCH_SIZE: "16" | |
| run: | | |
| set +e | |
| python main.py daily_job | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Launch Lightning inference studio | |
| id: launch_lightning_inference | |
| if: >- | |
| ${{ | |
| always() && | |
| !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && | |
| ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && | |
| steps.install_lightning_deps.outcome == 'success' && | |
| steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && | |
| steps.plan_ai_runtime.outputs.has_inference_url == 'true' && | |
| steps.plan_ai_runtime.outputs.enable_lightning_fallback == 'true' && | |
| ( | |
| (steps.warm_modal_inference.outcome == 'failure' || steps.verify_modal_predict.outcome == 'failure') || | |
| (steps.run_ai_bot_modal.outputs.rc != '0' && steps.run_ai_bot_modal_retry.outputs.rc != '0') | |
| ) | |
| }} | |
| continue-on-error: true | |
| timeout-minutes: 60 | |
| env: | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_CPU_THREADS: "8" | |
| LIGHTNING_INFERENCE_COMPUTE_NAME: ${{ steps.plan_ai_runtime.outputs.lightning_compute_name }} | |
| LIGHTNING_INFERENCE_DISK_GB: ${{ steps.plan_ai_runtime.outputs.lightning_disk_gb }} | |
| run: | | |
| python launch_lightning_inference_studio.py \ | |
| --config quant_platform/configs/lightning_inference_studio.yaml \ | |
| --allow-create \ | |
| --skip-public-health-check \ | |
| --status-out lightning-inference-status-local.json | |
| - name: Launch Lightning inference studio (manual full-studio mode) | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio }} | |
| env: | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TRAINED_MODEL_CPU_THREADS: "4" | |
| run: | | |
| python launch_lightning_inference_studio.py \ | |
| --config quant_platform/configs/lightning_inference_studio.yaml \ | |
| --allow-create \ | |
| --skip-public-health-check \ | |
| --status-out lightning-inference-status-local.json | |
| - name: Run AI Trading Bot in Lightning Studio | |
| id: run_ai_bot_in_lightning_studio | |
| if: >- | |
| ${{ | |
| always() && | |
| !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && | |
| ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && | |
| steps.install_lightning_deps.outcome == 'success' && | |
| steps.launch_lightning_inference.outcome == 'success' && | |
| steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && | |
| steps.plan_ai_runtime.outputs.enable_lightning_fallback == 'true' && | |
| ( | |
| (steps.warm_modal_inference.outcome == 'failure' || steps.verify_modal_predict.outcome == 'failure') || | |
| (steps.run_ai_bot_modal.outputs.rc != '0' && steps.run_ai_bot_modal_retry.outputs.rc != '0') | |
| ) | |
| }} | |
| continue-on-error: true | |
| timeout-minutes: 90 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_AI_API_KEY: ${{ secrets.ALPACA_AI_API_KEY }} | |
| ALPACA_AI_SECRET_KEY: ${{ secrets.ALPACA_AI_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_CORE_TRADING: "1" | |
| AI_RUNTIME_MODE: "full" | |
| AI_PRIMARY_BACKEND: "lightning-studio" | |
| AI_ROUTER_REASON: ${{ steps.plan_ai_runtime.outputs.reason }} | |
| AI_PROMPT_CANDIDATES_LIMIT: "200" | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "300" | |
| TRAINED_MODEL_READY_TIMEOUT_SECONDS: "1200" | |
| TRAINED_MODEL_READY_POLL_SECONDS: "15" | |
| TRAINED_MODEL_MAX_RETRIES: "0" | |
| TRAINED_MODEL_BATCH_SIZE: "4" | |
| TRAINED_MODEL_WARMUP_TIMEOUT_SECONDS: "1800" | |
| TRAINED_MODEL_CPU_THREADS: "8" | |
| TRAINED_MODEL_CLASS_TOKEN_INFERENCE: "1" | |
| run: | | |
| set +e | |
| python run_daily_job_lightning_studio.py \ | |
| --config quant_platform/configs/lightning_inference_studio.yaml \ | |
| --result-out results/daily_job_lightning_studio.json \ | |
| --session-name trading-bot-ai-daily-job-${{ github.run_id }} \ | |
| --timeout-seconds 5100 | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Run AI Trading Bot (Distilled Local Fallback) | |
| id: run_ai_bot_distilled_local | |
| if: >- | |
| ${{ | |
| always() && | |
| steps.install_base_dependencies.outcome == 'success' && | |
| !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && | |
| ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && | |
| ( | |
| steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local' || | |
| ( | |
| steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && | |
| steps.plan_ai_runtime.outputs.has_inference_url == 'true' && | |
| steps.plan_ai_runtime.outputs.enable_lightning_fallback == 'true' && | |
| ( | |
| (steps.warm_modal_inference.outcome == 'failure' || steps.verify_modal_predict.outcome == 'failure') || | |
| (steps.run_ai_bot_modal.outputs.rc != '0' && steps.run_ai_bot_modal_retry.outputs.rc != '0') | |
| ) && | |
| ( | |
| steps.launch_lightning_inference.outcome == 'failure' || | |
| steps.run_ai_bot_in_lightning_studio.outputs.rc != '0' | |
| ) | |
| ) | |
| ) | |
| }} | |
| continue-on-error: true | |
| timeout-minutes: 60 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_AI_API_KEY: ${{ secrets.ALPACA_AI_API_KEY }} | |
| ALPACA_AI_SECRET_KEY: ${{ secrets.ALPACA_AI_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_CORE_TRADING: "1" | |
| AI_RUNTIME_MODE: "distilled_local" | |
| AI_PRIMARY_BACKEND: "distilled-local" | |
| AI_ROUTER_REASON: "distilled_after_modal_failure" | |
| run: | | |
| set +e | |
| python quant_platform/scripts/prepare_ai_only_retry.py --results-dir results | |
| prep_rc=$? | |
| if [ "${prep_rc}" -ne 0 ]; then | |
| echo "rc=${prep_rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| fi | |
| python main.py daily_job | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Run AI Trading Bot (Emergency Distilled Retry) | |
| id: run_ai_bot_distilled_emergency_retry | |
| if: >- | |
| ${{ | |
| always() && | |
| steps.install_base_dependencies.outcome == 'success' && | |
| !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && | |
| ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && | |
| steps.run_ai_bot_distilled_local.conclusion == 'success' && | |
| steps.run_ai_bot_distilled_local.outputs.rc != '0' | |
| }} | |
| continue-on-error: true | |
| timeout-minutes: 60 | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_AI_API_KEY: ${{ secrets.ALPACA_AI_API_KEY }} | |
| ALPACA_AI_SECRET_KEY: ${{ secrets.ALPACA_AI_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_CORE_TRADING: "1" | |
| AI_RUNTIME_MODE: "distilled_local" | |
| AI_PRIMARY_BACKEND: "distilled-local" | |
| AI_ROUTER_REASON: "distilled_emergency_retry_after_failure" | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "450" | |
| TRAINED_MODEL_MAX_RETRIES: "1" | |
| run: | | |
| set +e | |
| python quant_platform/scripts/prepare_ai_only_retry.py --results-dir results | |
| prep_rc=$? | |
| if [ "${prep_rc}" -ne 0 ]; then | |
| echo "rc=${prep_rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| fi | |
| python main.py daily_job | |
| rc=$? | |
| echo "rc=${rc}" >> "${GITHUB_OUTPUT}" | |
| exit 0 | |
| - name: Send AI Failure Report | |
| id: send_ai_failure_report | |
| if: >- | |
| ${{ | |
| always() && | |
| !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) && | |
| ((github.event_name != 'workflow_dispatch') || !inputs.disable_ai_trading) && | |
| ( | |
| ( | |
| steps.plan_ai_runtime.outputs.runtime_mode == 'modal_full' && | |
| steps.plan_ai_runtime.outputs.has_inference_url == 'true' && | |
| ( | |
| (steps.warm_modal_inference.outcome == 'failure' || steps.verify_modal_predict.outcome == 'failure') || | |
| (steps.run_ai_bot_modal.outputs.rc != '0' && steps.run_ai_bot_modal_retry.outputs.rc != '0') | |
| ) && | |
| steps.run_ai_bot_distilled_local.outputs.rc != '0' && | |
| steps.run_ai_bot_distilled_emergency_retry.outputs.rc != '0' | |
| ) || | |
| ( | |
| steps.plan_ai_runtime.outputs.runtime_mode == 'distilled_local' && | |
| steps.run_ai_bot_distilled_local.outputs.rc != '0' && | |
| steps.run_ai_bot_distilled_emergency_retry.outputs.rc != '0' | |
| ) | |
| ) | |
| }} | |
| continue-on-error: true | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} | |
| AI_RUNTIME_MODE: ${{ steps.plan_ai_runtime.outputs.runtime_mode }} | |
| AI_RUNTIME_REASON: ${{ steps.plan_ai_runtime.outputs.reason }} | |
| LAUNCH_LIGHTNING_OUTCOME: ${{ steps.launch_lightning_inference.outcome }} | |
| RUN_AI_STUDIO_RC: ${{ steps.run_ai_bot_in_lightning_studio.outputs.rc }} | |
| WARM_MODAL_OUTCOME: ${{ steps.warm_modal_inference.outcome }} | |
| VERIFY_MODAL_OUTCOME: ${{ steps.verify_modal_predict.outcome }} | |
| RUN_AI_MODAL_RC: ${{ steps.run_ai_bot_modal.outputs.rc }} | |
| RUN_AI_MODAL_RETRY_RC: ${{ steps.run_ai_bot_modal_retry.outputs.rc }} | |
| RUN_AI_DISTILLED_RC: ${{ steps.run_ai_bot_distilled_local.outputs.rc }} | |
| RUN_AI_DISTILLED_RETRY_RC: ${{ steps.run_ai_bot_distilled_emergency_retry.outputs.rc }} | |
| run: | | |
| source_step="Run AI Trading Bot" | |
| detail="AI daily job failed before sending its report email. Run: ${RUN_URL}" | |
| if [ "${AI_RUNTIME_MODE}" = "modal_full" ]; then | |
| source_step="Run AI Trading Bot (Distilled Local Fallback)" | |
| detail="AI Modal full-model runtime failed and the distilled fallback also failed. Run: ${RUN_URL}" | |
| else | |
| source_step="Run AI Trading Bot (Distilled Local Fallback)" | |
| detail="AI distilled fallback runtime failed (${AI_RUNTIME_REASON}). Run: ${RUN_URL}" | |
| fi | |
| python send_failure_report.py \ | |
| --strategy-tag AI \ | |
| --source "${source_step}" \ | |
| --message "${detail}" \ | |
| --model-used "quant-trained-trading-model" | |
| - name: Run Trading Bot in Lightning Studio | |
| if: ${{ github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio }} | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| TRAINED_MODEL_API_KEY: ${{ secrets.TRAINED_MODEL_API_KEY }} | |
| TWELVEDATA_API_KEYS: ${{ secrets.TWELVEDATA_API_KEYS }} | |
| ARM_LIVE_TRADING: ${{ secrets.ARM_LIVE_TRADING }} | |
| ALPACA_AI_API_KEY: ${{ secrets.ALPACA_AI_API_KEY }} | |
| ALPACA_AI_SECRET_KEY: ${{ secrets.ALPACA_AI_SECRET_KEY }} | |
| LOG_LEVEL: ${{ secrets.LOG_LEVEL }} | |
| DISABLE_CORE_TRADING: "1" | |
| RESET_AI_POSITIONS: ${{ github.event_name == 'workflow_dispatch' && inputs.reset_ai_positions || 'false' }} | |
| TRAINED_MODEL_TIMEOUT_SECONDS: "300" | |
| TRAINED_MODEL_READY_TIMEOUT_SECONDS: "1200" | |
| TRAINED_MODEL_READY_POLL_SECONDS: "15" | |
| TRAINED_MODEL_MAX_RETRIES: "0" | |
| TRAINED_MODEL_BATCH_SIZE: "4" | |
| TRAINED_MODEL_WARMUP_TIMEOUT_SECONDS: "1800" | |
| TRAINED_MODEL_CPU_THREADS: "4" | |
| TRAINED_MODEL_CLASS_TOKEN_INFERENCE: "1" | |
| AI_PROMPT_CANDIDATES_LIMIT: "200" | |
| run: | | |
| extra_args="" | |
| if [ "${RESET_AI_POSITIONS}" = "true" ]; then | |
| extra_args="--reset-ai-positions" | |
| fi | |
| python run_daily_job_lightning_studio.py \ | |
| --config quant_platform/configs/lightning_inference_studio.yaml \ | |
| --result-out results/daily_job_lightning_studio.json \ | |
| --session-name trading-bot-daily-job-${{ github.run_id }} \ | |
| --timeout-seconds 5100 \ | |
| ${extra_args} | |
| - name: Generate Project Storage Report | |
| if: ${{ always() }} | |
| continue-on-error: true | |
| run: | | |
| mkdir -p results | |
| python quant_platform/scripts/report_project_storage.py --json-out results/project_storage_report.json | |
| - name: Upload artifacts (logs/reports) | |
| if: always() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: bot-results | |
| path: | | |
| results/*.csv | |
| results/*.json | |
| *.log | |
| lightning-inference-status-local.json | |
| retention-days: 7 | |
| - name: Save Data and Models Cache | |
| if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) }} | |
| uses: actions/cache/save@v4 | |
| with: | |
| path: | | |
| data/ | |
| models/ | |
| model_registry.json | |
| results/ | |
| key: ${{ runner.os }}-trading-bot-${{ github.run_id }} | |
| - name: Ensure Daily Email Coverage | |
| if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) }} | |
| continue-on-error: true | |
| env: | |
| SMTP_SERVER: ${{ secrets.SMTP_SERVER }} | |
| SMTP_PORT: ${{ secrets.SMTP_PORT }} | |
| SENDER_EMAIL: ${{ secrets.SENDER_EMAIL }} | |
| SENDER_PASSWORD: ${{ secrets.SENDER_PASSWORD }} | |
| RECIPIENT_EMAIL: rohan.santhoshkumar1@gmail.com | |
| RUN_URL: ${{ format('{0}/{1}/actions/runs/{2}', github.server_url, github.repository, github.run_id) }} | |
| CORE_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_core_trading) }} | |
| AI_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} | |
| BASE_DEPS_OUTCOME: ${{ steps.install_base_dependencies.outcome }} | |
| CORE_RC: ${{ steps.run_core_bot.outputs.rc }} | |
| CORE_RETRY_RC: ${{ steps.run_core_bot_retry.outputs.rc }} | |
| AI_RUNTIME_MODE: ${{ steps.plan_ai_runtime.outputs.runtime_mode }} | |
| RUN_AI_MODAL_RC: ${{ steps.run_ai_bot_modal.outputs.rc }} | |
| RUN_AI_MODAL_RETRY_RC: ${{ steps.run_ai_bot_modal_retry.outputs.rc }} | |
| RUN_AI_STUDIO_RC: ${{ steps.run_ai_bot_in_lightning_studio.outputs.rc }} | |
| RUN_AI_DISTILLED_RC: ${{ steps.run_ai_bot_distilled_local.outputs.rc }} | |
| RUN_AI_DISTILLED_RETRY_RC: ${{ steps.run_ai_bot_distilled_emergency_retry.outputs.rc }} | |
| AI_FAILURE_REPORT_OUTCOME: ${{ steps.send_ai_failure_report.outcome }} | |
| run: | | |
| core_ok="false" | |
| ai_ok="false" | |
| if [ "${CORE_EXPECTED}" = "false" ]; then | |
| core_ok="true" | |
| elif [ "${CORE_RC}" = "0" ] || [ "${CORE_RETRY_RC}" = "0" ]; then | |
| core_ok="true" | |
| fi | |
| if [ "${AI_EXPECTED}" = "false" ]; then | |
| ai_ok="true" | |
| elif [ "${AI_RUNTIME_MODE}" = "modal_full" ] && { [ "${RUN_AI_MODAL_RC}" = "0" ] || [ "${RUN_AI_MODAL_RETRY_RC}" = "0" ] || [ "${RUN_AI_STUDIO_RC}" = "0" ] || [ "${RUN_AI_DISTILLED_RC}" = "0" ] || [ "${RUN_AI_DISTILLED_RETRY_RC}" = "0" ]; }; then | |
| ai_ok="true" | |
| elif [ "${AI_RUNTIME_MODE}" = "distilled_local" ] && { [ "${RUN_AI_DISTILLED_RC}" = "0" ] || [ "${RUN_AI_DISTILLED_RETRY_RC}" = "0" ]; }; then | |
| ai_ok="true" | |
| fi | |
| if [ "${BASE_DEPS_OUTCOME}" != "success" ]; then | |
| # Base dependency failure reports are emitted earlier; do not duplicate here. | |
| exit 0 | |
| fi | |
| if [ "${core_ok}" != "true" ]; then | |
| python send_failure_report.py \ | |
| --strategy-tag Core \ | |
| --source "Email Coverage Guard" \ | |
| --message "Core report email was not confirmed from explicit run return codes. Run: ${RUN_URL}" | |
| fi | |
| if [ "${ai_ok}" != "true" ]; then | |
| if [ "${AI_FAILURE_REPORT_OUTCOME}" != "success" ]; then | |
| python send_failure_report.py \ | |
| --strategy-tag AI \ | |
| --source "Email Coverage Guard" \ | |
| --message "AI report email was not confirmed from explicit run return codes (fallback may also have failed). Run: ${RUN_URL}" \ | |
| --model-used "quant-trained-trading-model" | |
| fi | |
| fi | |
| - name: Evaluate Scheduled/Runner Run Status | |
| if: ${{ always() && !(github.event_name == 'workflow_dispatch' && inputs.run_in_lightning_studio) }} | |
| env: | |
| BASE_DEPS_OUTCOME: ${{ steps.install_base_dependencies.outcome }} | |
| CORE_RC: ${{ steps.run_core_bot.outputs.rc }} | |
| CORE_RETRY_RC: ${{ steps.run_core_bot_retry.outputs.rc }} | |
| CORE_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_core_trading) }} | |
| AI_EXPECTED: ${{ (github.event_name != 'workflow_dispatch') || (github.event_name == 'workflow_dispatch' && !inputs.disable_ai_trading) }} | |
| INSTALL_LIGHTNING_OUTCOME: ${{ steps.install_lightning_deps.outcome }} | |
| AI_RUNTIME_MODE: ${{ steps.plan_ai_runtime.outputs.runtime_mode }} | |
| LAUNCH_LIGHTNING_OUTCOME: ${{ steps.launch_lightning_inference.outcome }} | |
| RUN_AI_STUDIO_RC: ${{ steps.run_ai_bot_in_lightning_studio.outputs.rc }} | |
| WARM_MODAL_OUTCOME: ${{ steps.warm_modal_inference.outcome }} | |
| VERIFY_MODAL_OUTCOME: ${{ steps.verify_modal_predict.outcome }} | |
| RUN_AI_MODAL_RC: ${{ steps.run_ai_bot_modal.outputs.rc }} | |
| RUN_AI_MODAL_RETRY_RC: ${{ steps.run_ai_bot_modal_retry.outputs.rc }} | |
| RUN_AI_DISTILLED_RC: ${{ steps.run_ai_bot_distilled_local.outputs.rc }} | |
| RUN_AI_DISTILLED_RETRY_RC: ${{ steps.run_ai_bot_distilled_emergency_retry.outputs.rc }} | |
| run: | | |
| rc=0 | |
| if [ "${BASE_DEPS_OUTCOME}" != "success" ]; then | |
| echo "Base dependency install failed: ${BASE_DEPS_OUTCOME}" | |
| rc=1 | |
| fi | |
| if [ "${BASE_DEPS_OUTCOME}" = "success" ] && [ "${CORE_EXPECTED}" = "true" ] && [ "${CORE_RC}" != "0" ] && [ "${CORE_RETRY_RC}" != "0" ]; then | |
| echo "Core path failed: initial_rc=${CORE_RC}, retry_rc=${CORE_RETRY_RC}" | |
| rc=1 | |
| fi | |
| if [ "${BASE_DEPS_OUTCOME}" = "success" ] && [ "${AI_EXPECTED}" = "true" ]; then | |
| if [ "${AI_RUNTIME_MODE}" = "modal_full" ]; then | |
| modal_failed="false" | |
| if [ "${WARM_MODAL_OUTCOME}" != "success" ]; then | |
| echo "Modal warmup failed: ${WARM_MODAL_OUTCOME}" | |
| modal_failed="true" | |
| fi | |
| if [ "${WARM_MODAL_OUTCOME}" = "success" ] && [ "${VERIFY_MODAL_OUTCOME}" != "success" ]; then | |
| echo "Modal predict preflight failed: ${VERIFY_MODAL_OUTCOME}" | |
| modal_failed="true" | |
| fi | |
| if [ "${WARM_MODAL_OUTCOME}" = "success" ] && [ "${VERIFY_MODAL_OUTCOME}" = "success" ] && [ "${RUN_AI_MODAL_RC}" != "0" ] && [ "${RUN_AI_MODAL_RETRY_RC}" != "0" ]; then | |
| echo "AI Modal run failed: initial_rc=${RUN_AI_MODAL_RC}, retry_rc=${RUN_AI_MODAL_RETRY_RC}" | |
| modal_failed="true" | |
| fi | |
| if [ "${modal_failed}" = "true" ] && [ "${RUN_AI_STUDIO_RC}" != "0" ] && [ "${RUN_AI_DISTILLED_RC}" != "0" ] && [ "${RUN_AI_DISTILLED_RETRY_RC}" != "0" ]; then | |
| echo "AI fallback also failed: studio_rc=${RUN_AI_STUDIO_RC}, distilled_rc=${RUN_AI_DISTILLED_RC}, retry_rc=${RUN_AI_DISTILLED_RETRY_RC}" | |
| rc=1 | |
| fi | |
| else | |
| if [ "${RUN_AI_DISTILLED_RC}" != "0" ] && [ "${RUN_AI_DISTILLED_RETRY_RC}" != "0" ]; then | |
| echo "AI distilled fallback run failed: rc=${RUN_AI_DISTILLED_RC}, retry_rc=${RUN_AI_DISTILLED_RETRY_RC}" | |
| rc=1 | |
| fi | |
| fi | |
| fi | |
| exit "${rc}" |