pipelined_ingestion_compute.yaml

cloud_id: {{env["ANYSCALE_CLOUD_ID"]}}
region: us-west-2

max_workers: 999

aws:
    BlockDeviceMappings:
        - DeviceName: /dev/sda1
          Ebs:
            DeleteOnTermination: true
            VolumeSize: 500

head_node_type:
    name: head_node
    instance_type: m6i.8xlarge

worker_node_types:
    - name: memory_node
      instance_type: m6i.8xlarge
      min_workers: 20
      max_workers: 20
      use_spot: false
    - name: gpu_node
      instance_type: m6i.8xlarge
      min_workers: 4
      max_workers: 4
      use_spot: false
      resources:
        cpu: 0
        gpu: 4