Skip to content

Commit 7dc0233

Browse files
alabdaothetechnocrat-dev
alabdao
and
thetechnocrat-dev
authored
[LAB-636] ci setup (#679)
Co-authored-by: thetechnocrat-dev <josh.mcmenemy@openzyme.bio>
1 parent f808ec8 commit 7dc0233

File tree

17 files changed

+653
-75
lines changed

17 files changed

+653
-75
lines changed

‎.github/workflows/ci.yml

Lines changed: 234 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,234 @@
1+
# This workflow will build a golang project and runs CI tests
2+
3+
name: CI
4+
5+
on:
6+
push:
7+
branches: ["main"]
8+
paths-ignore:
9+
- 'docs/**'
10+
- 'infrastructure/**'
11+
pull_request:
12+
branches: ["main"]
13+
paths-ignore:
14+
- 'docs/**'
15+
- 'infrastructure/**'
16+
17+
# Only run one at a time
18+
concurrency:
19+
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
20+
cancel-in-progress: true
21+
22+
jobs:
23+
go:
24+
runs-on: ubuntu-22.04
25+
environment: ci
26+
steps:
27+
- uses: actions/checkout@v3
28+
29+
- name: Set up Go
30+
uses: actions/setup-go@v3
31+
with:
32+
go-version: 1.20.3
33+
34+
- name: Install dependencies
35+
run: go mod download
36+
37+
- name: Build
38+
run: go build
39+
40+
- name: upload plex binary to be used by other jobs
41+
uses: actions/upload-artifact@v3
42+
with:
43+
name: plex-binary
44+
path: ./plex
45+
46+
- name: Test
47+
run: go test ./... -v
48+
49+
ci-setup-compose:
50+
runs-on: ubuntu-22.04
51+
environment: ci
52+
steps:
53+
- name: Download docker compose
54+
run: |
55+
# Download docker-compose plugin
56+
curl -sSL https://github.com/docker/compose/releases/download/v2.20.3/docker-compose-linux-x86_64 -o docker-compose
57+
58+
- name: upload docker-compose plugin to be used later
59+
uses: actions/upload-artifact@v3
60+
with:
61+
name: docker-compose-plugin
62+
path: ./docker-compose
63+
64+
ci-setup-ipfs:
65+
runs-on: ubuntu-22.04
66+
environment: ci
67+
steps:
68+
- name: Download and extract ipfs binary tarball
69+
run: |
70+
# Download ipfs binary
71+
curl -sSL https://github.com/ipfs/kubo/releases/download/v0.18.0/kubo_v0.18.0_linux-amd64.tar.gz -o kubo.tgz
72+
tar -zxvf kubo.tgz
73+
74+
- name: upload ipfs binary to be used later
75+
uses: actions/upload-artifact@v3
76+
with:
77+
name: ipfs-binary
78+
path: ./kubo/ipfs
79+
80+
ci-public:
81+
needs:
82+
- go
83+
- ci-setup-compose
84+
runs-on: ubuntu-22.04
85+
environment: ci
86+
steps:
87+
- name: Checkout code
88+
uses: actions/checkout@v4
89+
90+
- name: download plex binary
91+
uses: actions/download-artifact@v3
92+
with:
93+
name: plex-binary
94+
95+
- name: download docker compose plugin artifact
96+
uses: actions/download-artifact@v3
97+
with:
98+
name: docker-compose-plugin
99+
100+
- name: Setup docker compose plugin
101+
run: |
102+
# Install docker-compose
103+
DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
104+
mkdir -p $DOCKER_CONFIG/cli-plugins
105+
mv docker-compose $DOCKER_CONFIG/cli-plugins/docker-compose
106+
chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
107+
108+
# Output version info
109+
docker version
110+
docker compose version
111+
112+
- name: docker compose build
113+
run: |
114+
# Build in parallel
115+
docker compose build --parallel
116+
117+
- name: Bring up the stack
118+
run: |
119+
# Setup docker compose private
120+
docker compose up -d
121+
122+
- name: Run Equibind
123+
run: |
124+
# Add execute permission
125+
chmod +x plex
126+
127+
# Run tests against it
128+
export BACALHAU_API_HOST=127.0.0.1
129+
130+
result_dir=$(./plex init -t tools/equibind.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true -a test -a ci | grep 'Finished processing, results written to' | sed -n 's/^.*Finished processing, results written to //p' | sed 's/\/io.json//')
131+
cd "$result_dir/entry-0/outputs"
132+
if [ "$(find . -name '*docked.sdf' | grep 'docked.sdf')" == "" ]; then
133+
echo "No docked files found"
134+
exit 1
135+
else
136+
echo "Docked files found:"
137+
find . -name '*docked.sdf' | grep 'docked.sdf'
138+
fi
139+
140+
- name: upload outputs
141+
uses: actions/upload-artifact@v3
142+
with:
143+
name: ci-public-output
144+
path: |
145+
job-*
146+
jobs/
147+
148+
ci-private:
149+
needs:
150+
- go
151+
- ci-setup-compose
152+
- ci-setup-ipfs
153+
runs-on: ubuntu-22.04
154+
environment: ci
155+
steps:
156+
- name: Checkout code
157+
uses: actions/checkout@v4
158+
159+
- name: download plex binary
160+
uses: actions/download-artifact@v3
161+
with:
162+
name: plex-binary
163+
164+
- name: download docker compose plugin artifact
165+
uses: actions/download-artifact@v3
166+
with:
167+
name: docker-compose-plugin
168+
169+
- name: download ipfs binary
170+
uses: actions/download-artifact@v3
171+
with:
172+
name: ipfs-binary
173+
174+
- name: Setup docker compose plugin
175+
run: |
176+
# Install docker-compose
177+
DOCKER_CONFIG=${DOCKER_CONFIG:-$HOME/.docker}
178+
mkdir -p $DOCKER_CONFIG/cli-plugins
179+
mv docker-compose $DOCKER_CONFIG/cli-plugins/docker-compose
180+
chmod +x $DOCKER_CONFIG/cli-plugins/docker-compose
181+
182+
# Output version info
183+
docker version
184+
docker compose version
185+
186+
- name: docker compose build
187+
run: |
188+
# Build in parallel
189+
docker compose build --parallel
190+
191+
- name: Bring up the stack
192+
run: |
193+
# Setup docker compose private
194+
docker compose -f docker-compose.yml -f docker-compose.private.yml up -d
195+
196+
- name: Run Equibind
197+
run: |
198+
set -x
199+
# Add execute permission
200+
chmod +x plex ipfs
201+
202+
# Run tests against it
203+
# using temp directory for ipfs stuff
204+
export IPFS_PATH=$(mktemp -d)
205+
206+
# Initialize IPFS repo
207+
./ipfs init -e
208+
209+
# Copy over swarm key and config
210+
cp -rav $(pwd)/docker/ipfs_data/* "${IPFS_PATH}/"
211+
212+
export BACALHAU_API_HOST=127.0.0.1
213+
export BACALHAU_SERVE_IPFS_PATH="${IPFS_PATH}"
214+
export BACALHAU_IPFS_SWARM_ADDRESSES="/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWLpoHJCGxxKozRaUK1e1m2ocyVPB9dzbsU2cydujYBCD7"
215+
216+
result_dir=$(./plex init -t tools/equibind.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true -a test -a ci | grep 'Finished processing, results written to' | sed -n 's/^.*Finished processing, results written to //p' | sed 's/\/io.json//')
217+
cd "$result_dir/entry-0/outputs"
218+
ls -ltraR
219+
cat exitCode stderr stderr log.txt || true
220+
if [ "$(find . -name '*docked.sdf' | grep 'docked.sdf')" == "" ]; then
221+
echo "No docked files found"
222+
exit 1
223+
else
224+
echo "Docked files found:"
225+
find . -name '*docked.sdf' | grep 'docked.sdf'
226+
fi
227+
228+
- name: upload outputs
229+
uses: actions/upload-artifact@v3
230+
with:
231+
name: ci-private-output
232+
path: |
233+
job-*
234+
jobs/

‎.github/workflows/go.yml

Lines changed: 0 additions & 50 deletions
This file was deleted.

‎.gitignore

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -326,4 +326,4 @@ gateway/gorm.db
326326
# Ignore tool specific files
327327
*zinc*
328328
*AF-*
329-
tools/tmp*
329+
tools/tmp*

‎README.md

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -31,9 +31,9 @@
3131

3232

3333
Plex is a simple client for distributed computation.
34-
* 🌎 **Build once, run anywhere:** Plex is using distributed compute and storage to run containers on a public network. Need GPUs? We got you covered.
34+
* 🌎 **Build once, run anywhere:** Plex is using distributed compute and storage to run containers on a public network. Need GPUs? We got you covered.
3535
* 🔍 **Content-addressed by default:** Every file processed by plex has a deterministic address based on its content. Keep track of your files and always share the right results with other scientists.
36-
* 🪙 **Ownernship tracking built-in** Every compute event on plex is mintable as an on-chain token that grants the holder rights over the newly generated data.
36+
* 🪙 **Ownernship tracking built-in** Every compute event on plex is mintable as an on-chain token that grants the holder rights over the newly generated data.
3737
* 🔗 **Strictly composable:** Every tool in plex has declared inputs and outputs. Plugging together tools by other authors should be easy.
3838

3939
Plex is based on [Bacalhau](https://www.bacalhau.org/), [IPFS](https://ipfs.tech/), and inspired by the [Common Workflow Language](https://www.commonwl.org/user_guide/introduction/quick-start.html).
@@ -67,7 +67,7 @@ Invoke-Expression (Invoke-WebRequest -Uri "https://raw.githubusercontent.com/lab
6767

6868
2. Submit an example plex job
6969
```
70-
./plex init -t tools/equibind.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true
70+
./plex init -t tools/equibind.json -i '{"protein": ["testdata/binding/abl/7n9g.pdb"], "small_molecule": ["testdata/binding/abl/ZINC000003986735.sdf"]}' --scatteringMethod=dotProduct --autoRun=true
7171
```
7272

7373
![Getting Started](./readme-getting-started-2x.gif)
@@ -107,14 +107,54 @@ POSTGRES_USER=labdao
107107
POSTGRES_DB=labdao
108108
POSTGRES_HOST=localhost
109109
```
110-
* Recommended: Install [direnv](https://direnv.net/). With it installed you can create `.env` file with the above environment variables and have them automagically set when you descend into the folder.
110+
* Recommended: Install [direnv](https://direnv.net/). With it installed you can create `.env` file with the above environment variables and have them automagically set when you descend into the folder.
111111

112-
# Start the database
112+
# Running complete stack locally
113+
We have `docker-compose` files available to bring up the stack locally.
114+
115+
Note:
116+
* Only `amd64` architecture is currently supported.
117+
* New docker installation include docker compose, older installations required you install docker-compose separately and run `docker-compose up -d`
118+
119+
## Running
113120
```
121+
# Optionally, build in parallel before running
122+
docker compose build --parallel
123+
124+
# Build and bring up stack
114125
docker compose up -d
115126
```
116127

117-
Note: New docker installation include docker compose, older installations required you install docker-compose separately and run `docker-compose up -d`
128+
To run `plex` cli against local environment simply set `BACALHAU_API_HOST=127.0.0.1`
129+
130+
131+
## Running with private IPFS
132+
> Requirement to have `ipfs` available locally.
133+
134+
```
135+
docker compose -f docker-compose.yml -f docker-compose.private.yml up -d
136+
```
137+
To run `plex` cli against local private environment `export` the following params to your shell before executing `plex` commands:
138+
```
139+
140+
# using temp directory for ipfs stuff
141+
IPFS_PATH=$(mktemp -d)
142+
143+
# Initialize IPFS repo
144+
ipfs init -e
145+
146+
# Copy over swarm key and config
147+
cp -rav $(pwd)/docker/ipfs_data/* "${IPFS_PATH}/"
148+
149+
export BACALHAU_SERVE_IPFS_PATH="${IPFS_PATH}"
150+
export BACALHAU_IPFS_SWARM_ADDRESSES="/ip4/127.0.0.1/tcp/4001/p2p/12D3KooWLpoHJCGxxKozRaUK1e1m2ocyVPB9dzbsU2cydujYBCD7"
151+
```
152+
153+
## Running backend database only
154+
```
155+
docker compose up -d dbbackend
156+
```
157+
118158

119159
# Start the Frontend React App
120160

@@ -154,12 +194,12 @@ export IPFS_CONNECT=/ip4/127.0.0.1/tcp/5001/p2p/<your id goes here>
154194
LOG_LEVEL=debug bacalhau serve --job-selection-accept-networked --limit-total-gpu 1 --limit-total-memory 12gb --ipfs-connect $IPFS_CONNECT
155195
```
156196

157-
To download large bacalhau results the below command may need ran
197+
To download large bacalhau results the below command may need ran
158198
```
159199
sudo sysctl -w net.core.rmem_max=2500000
160200
```
161201

162202
## 💁 Contributing
163-
PRs are welcome! Please consider our [Contribute Guidelines](https://docs.labdao.xyz/about-us/contributer_policy) when joining.
203+
PRs are welcome! Please consider our [Contribute Guidelines](https://docs.labdao.xyz/about-us/contributer_policy) when joining.
164204

165205
From time to time, we also post ```help-wanted``` bounty issues - please consider our [Bounty Policy](https://docs.labdao.xyz/about-us/bounty_policy) when engaging with LabDAO.

0 commit comments

Comments
 (0)