-
Notifications
You must be signed in to change notification settings - Fork 7
/
Makefile
152 lines (125 loc) · 5.81 KB
/
Makefile
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
include *.mk
include .envrc
## create cluster and install minio and prefect
kubes: cluster kubes-minio kubes-prefect
## create k3s cluster
cluster:
# port 4200 on the host is mapped to ingress on port 80
k3d cluster create prefect --registry-create prefect-registry:0.0.0.0:5550 \
-p 4200:80@loadbalancer -p 9000:9000@loadbalancer -p 9001:9001@loadbalancer \
-p 10001:10001@loadbalancer -p 8265:8265@loadbalancer -p 6379:6379@loadbalancer \
--wait
@echo "Probing until cluster is ready (~60 secs)..." && export KUBECONFIG=$$(k3d kubeconfig write prefect) && \
while ! kubectl get crd ingressroutes.traefik.containo.us 2> /dev/null ; do sleep 10 && echo $$((i=i+10)); done
@echo -e "\nTo use your cluster set:\n"
@echo "export KUBECONFIG=$$(k3d kubeconfig write prefect)"
## install minio
kubes-minio:
# root user and password is stored in the minio secret
helm upgrade --install --repo https://charts.bitnami.com/bitnami minio minio --set auth.rootUser=minioadmin --set auth.rootPassword=minioadmin \
--wait --debug > /dev/null
kubectl apply -f infra/lb-minio.yaml
kubectl exec deploy/minio -- mc mb -p local/minio-flows
## install kuberay operator using quickstart manifests
kubes-ray: KUBERAY_VERSION=v0.3.0
kubes-ray:
# install CRDs
kubectl apply --server-side -k "github.com/ray-project/kuberay/manifests/cluster-scope-resources?ref=${KUBERAY_VERSION}&timeout=90s"
# install kuberay operator
kubectl apply -k "github.com/ray-project/kuberay/manifests/base?ref=${KUBERAY_VERSION}&timeout=90s"
kubectl apply -f infra/ray-cluster.complete.yaml
kubectl apply -f infra/lb-ray.yaml
@echo -e "\nProbing for the ray cluster to be available (~3 mins)..." && \
while ! $(venv)/bin/python -m flows.ping_ray; do sleep 10; done
## install prefect server, worker and agent into kubes cluster
kubes-prefect:
kubectl apply -f infra/rbac-dask.yaml
kubectl apply -f infra/sa-flows.yaml
helm upgrade --install --repo https://prefecthq.github.io/prefect-helm \
prefect-server prefect-server --version=2024.4.18205353 \
--values infra/values-server.yaml --wait --debug > /dev/null
helm upgrade --install --repo https://prefecthq.github.io/prefect-helm \
prefect-worker prefect-worker --version=2024.4.18205353 \
--values infra/values-worker.yaml --wait --debug > /dev/null
helm upgrade --install --repo https://prefecthq.github.io/prefect-helm \
prefect-agent prefect-agent --version=2024.4.18205353 \
--values infra/values-agent.yaml --wait --debug > /dev/null
@echo -e "\nProbing for the prefect API to be available (~30 secs)..." && \
while ! curl -fsS http://localhost:4200/api/admin/version ; do sleep 5; done && echo
## restart prefect server (delete all flows)
server-restart:
kubectl rollout restart deploy/prefect-server
## delete objects in minio bucket
minio-empty:
kubectl exec deploy/minio -- mc rm local/minio-flows/ --recursive --force
## show the prefect job manifest
prefect-job-manifest:
prefect kubernetes manifest flow-run-job
## run parameterised flow
param-flow: $(venv)
$(venv)/bin/python -m flows.param_flow
## run dask flow
dask-flow: $(venv)
$(venv)/bin/python -m flows.dask_flow
## run ray flow
ray-flow: export PREFECT_LOCAL_STORAGE_PATH=/tmp/prefect/storage # see https://github.com/PrefectHQ/prefect-ray/issues/26
# PREFECT_API_URL needs to be accessible from the process running the flow and within the ray cluster
# to make this work locally, add 127.0.0.1 prefect-server to /etc/hosts TODO: find a better fix
ray-flow: $(venv)
$(venv)/bin/python -m flows.ray_flow
## run sub flow
sub-flow: $(venv)
$(venv)/bin/python -m flows.sub_flow
## build and push docker image
publish:
docker buildx bake --push
## deploy flows to run on kubernetes
deploy: $(venv) publish
# use minio as the s3 remote file system & deploy flows via python
set -e && . config/fsspec-env.sh && $(venv)/bin/python -m flows.deploy
# deploy flows via prefect.yaml
$(venv)/bin/prefect --no-prompt deploy --all
$(venv)/bin/prefect deployment ls
@echo Visit http://localhost:4200
## run deployments
run: $(venv)
$(venv)/bin/python -m flows.run
## start prefect ui
ui: $(venv)
PATH="$(venv)/bin:$$PATH" prefect server start
## show kube logs for the server and worker
kubes-logs:
kubectl logs -l "app.kubernetes.io/name in (prefect-server, prefect-worker, prefect-agent)" -f --tail=-1
## show kube logs for flows
kubes-logs-jobs:
kubectl logs -l "job-name" -f --tail=-1
## show kube logs for dask scheduler and workers
kubes-logs-dask:
kubectl logs -l "app=dask" -f --tail=-1
## show flow run logs
logs: assert-id
curl -H "Content-Type: application/json" -X POST --data '{"logs":{"flow_run_id":{"any_":["$(id)"]},"level":{"ge_":0}},"sort":"TIMESTAMP_ASC"}' -s "http://localhost:4200/api/logs/filter" | jq -r '.[] | [.timestamp,.level,.message] |@tsv'
## show flow runs
flow-runs:
$(venv)/bin/prefect flow-run ls
## access .db in kubes
kubes-db:
kubectl exec -i -t deploy/prefect-server -- /bin/bash -c 'hash sqlite3 || (apt-get update && apt-get install sqlite3) && sqlite3 ~/.prefect/prefect.db'
## upgrade to latest version of prefect
upgrade:
latest=$$(PIP_REQUIRE_VIRTUALENV=false pip index versions prefect | tail -n +1 | head -n1 | sed -E 's/.*\(([0-9.]+)\)/\1/') && \
rg -l 2.18.0 | xargs sed -i '' "s/2.18.0/$$latest/g"
make install
## forward traefik dashboard
tdashboard:
@echo Forwarding traefik dashboard to http://localhost:8999/dashboard/
tpod=$$(kubectl get pod -n kube-system -l app.kubernetes.io/name=traefik -o custom-columns=:metadata.name --no-headers=true) && \
kubectl -n kube-system port-forward $$tpod 8999:9000
## inspect block document
api-block-doc: assert-id
@curl -s "http://localhost:4200/api/block_documents/$(id)" | jq -r 'if .message then .message else {data, block_type:{ name: .block_type.name }} end'
assert-id:
ifndef id
@echo Missing id variable, eg: make $(MAKECMDGOALS) id=3af1d9d7-d52b-4251-87a0-dfe9c82daa3f
@exit 42
endif