pull_intraday_per_minute.yml

apiVersion: batch/v1
kind: Job
metadata:
  name: ae-dataset-intraday
  annotations:
    description: The Stock Analysis Dataset Collector for fetching from IEX Cloud and Tradier datasets every minute
    runtime: python3
  labels:
    sa: engine
    purpose: collect-intraday
    layer: backend
    messaging: redis
    cache: redis
    pubsub: publisher
spec:
  template:
    metadata:
      labels:
        app: ae-dataset-intraday
        backend: enabled
        purpose: stock-analysis
        layer: backend
        messaging: redis
        cache: redis
        pubsub: publisher
    spec:
      hostname: ae-dataset-intraday
      restartPolicy: Never
      # imagePullSecrets:
      # - name: ae.docker.creds
      containers:
      - image: jayjohnson/stock-analysis-engine:latest
        imagePullPolicy: Always
        name: ae-dataset-intraday
        resources: {}
        command: [
          "/bin/bash",
          "-c",
          "cd /opt/sa/ && . /opt/venv/bin/activate && /opt/sa/tools/start-dataset-collection.sh"
        ]
        env:
        - name: DATASET_SOURCES
          value: intraday
        - name: POSTGRES_USER
          valueFrom:
            secretKeyRef:
              name: api.db
              key: username
        - name: POSTGRES_PASSWORD
          valueFrom:
            secretKeyRef:
              name: api.db
              key: password
        - name: POSTGRES_DB
          valueFrom:
            secretKeyRef:
              name: api.db
              key: dbname
        - name: S3_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: ae.k8.minio.s3
              key: aws_access_key_id
        - name: S3_SECRET_KEY
          valueFrom:
            secretKeyRef:
              name: ae.k8.minio.s3
              key: aws_secret_access_key
        - name: S3_ADDRESS
          value: minio-service:9000
        - name: AWS_ACCESS_KEY_ID
          valueFrom:
            secretKeyRef:
              name: ae.k8.minio.s3
              key: aws_access_key_id
        - name: AWS_SECRET_ACCESS_KEY
          valueFrom:
            secretKeyRef:
              name: ae.k8.minio.s3
              key: aws_secret_access_key
        - name: S3_REGION_NAME
          value: us-east-1
        - name: WORKER_BROKER_URL
          value: redis://redis-master:6379/13
        - name: WORKER_BACKEND_URL
          value: redis://redis-master:6379/14
        - name: REDIS_ADDRESS
          value: redis-master:6379
        - name: REDIS_DB
          value: "0"
        # Stock Analysis - Dataset Collector Envs:
        - name: S3_BUCKET
          value: pricing
        - name: DAILY_S3_BUCKET_NAME
          value: daily
        - name: MINUTE_S3_BUCKET_NAME
          value: minute
        - name: TICK_S3_BUCKET_NAME
          value: tick
        - name: STATS_S3_BUCKET_NAME
          value: stats
        - name: PEERS_S3_BUCKET_NAME
          value: peers
        - name: NEWS_S3_BUCKET_NAME
          value: news
        - name: FINANCIALS_S3_BUCKET_NAME
          value: financials
        - name: EARNINGS_S3_BUCKET_NAME
          value: earnings
        - name: DIVIDENDS_S3_BUCKET_NAME
          value: dividends
        - name: COMPANY_S3_BUCKET_NAME
          value: company
        - name: PREPARE_S3_BUCKET_NAME
          value: prepared
        - name: ANALYZE_S3_BUCKET_NAME
          value: analyzed
        - name: TICKER
          value: SPY
        # Pull your own tickers here:
        - name: DEFAULT_TICKERS
          value: SPY,AMZN,TSLA
        # set to your Slack webhook url:
        - name: SLACK_WEBHOOK
          valueFrom:
            secretKeyRef:
              name: ae.k8.slack
              key: webhook
          # set to your Slack OAuth Access Token or Bot User OAuth Access Token
        - name: SLACK_ACCESS_TOKEN
          valueFrom:
            secretKeyRef:
              name: ae.k8.slack
              key: access_token
          # set comma separated Slack channels (without '#') to publish plots to
        - name: SLACK_PUBLISH_PLOT_CHANNELS
          valueFrom:
            secretKeyRef:
              name: ae.k8.slack
              key: channels
        # set to "1" to enable publishing to slack when
        # each ticker's job completes
        - name: DATASET_COLLECTION_SLACK_ALERTS
          value: "0"
        # set to "1" to publish Celery task failures to Slack
        - name: PROD_SLACK_ALERTS
          value: "0"