Skip to content

Commit 388229d

Browse files
eyurtseveyurtsev
authored
Set up CI (#2)
Basic CI set up
1 parent 744758e commit 388229d

File tree

7 files changed

+2506
-97
lines changed

7 files changed

+2506
-97
lines changed

‎.github/actions/poetry_setup/action.yml‎

Lines changed: 91 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,91 @@
1+
# An action for setting up poetry install with caching.
2+
# Using a custom action since the default action does not
3+
# take poetry install groups into account.
4+
# Action code from:
5+
# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
6+
name: poetry-install-with-caching
7+
description: Poetry install with support for caching of dependency groups.
8+
9+
inputs:
10+
python-version:
11+
description: Python version, supporting MAJOR.MINOR only
12+
required: true
13+
14+
poetry-version:
15+
description: Poetry version
16+
required: true
17+
18+
cache-key:
19+
description: Cache key to use for manual handling of caching
20+
required: true
21+
22+
working-directory:
23+
description: Directory whose poetry.lock file should be cached
24+
required: true
25+
26+
runs:
27+
using: composite
28+
steps:
29+
- uses: actions/setup-python@v4
30+
name: Setup python ${{ inputs.python-version }}
31+
with:
32+
python-version: ${{ inputs.python-version }}
33+
34+
- uses: actions/cache@v3
35+
id: cache-bin-poetry
36+
name: Cache Poetry binary - Python ${{ inputs.python-version }}
37+
env:
38+
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
39+
with:
40+
path: |
41+
/opt/pipx/venvs/poetry
42+
# This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
43+
key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}
44+
45+
- name: Refresh shell hashtable and fixup softlinks
46+
if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
47+
shell: bash
48+
env:
49+
POETRY_VERSION: ${{ inputs.poetry-version }}
50+
PYTHON_VERSION: ${{ inputs.python-version }}
51+
run: |
52+
set -eux
53+
54+
# Refresh the shell hashtable, to ensure correct `which` output.
55+
hash -r
56+
57+
# `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
58+
# Delete and recreate the softlinks pipx expects to have.
59+
rm /opt/pipx/venvs/poetry/bin/python
60+
cd /opt/pipx/venvs/poetry/bin
61+
ln -s "$(which "python$PYTHON_VERSION")" python
62+
chmod +x python
63+
cd /opt/pipx_bin/
64+
ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
65+
chmod +x poetry
66+
67+
# Ensure everything got set up correctly.
68+
/opt/pipx/venvs/poetry/bin/python --version
69+
/opt/pipx_bin/poetry --version
70+
71+
- name: Install poetry
72+
if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
73+
shell: bash
74+
env:
75+
POETRY_VERSION: ${{ inputs.poetry-version }}
76+
PYTHON_VERSION: ${{ inputs.python-version }}
77+
run: pipx install "poetry==$POETRY_VERSION" --python "python$PYTHON_VERSION" --verbose
78+
79+
- name: Restore pip and poetry cached dependencies
80+
uses: actions/cache@v3
81+
env:
82+
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
83+
WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
84+
with:
85+
path: |
86+
~/.cache/pip
87+
~/.cache/pypoetry/virtualenvs
88+
~/.cache/pypoetry/cache
89+
~/.cache/pypoetry/artifacts
90+
${{ env.WORKDIR }}/.venv
91+
key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}

‎.github/workflows/_lint.yml‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
name: lint
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
working-directory:
7+
required: true
8+
type: string
9+
description: "From which folder this pipeline executes"
10+
11+
env:
12+
POETRY_VERSION: "1.7.1"
13+
WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
14+
15+
jobs:
16+
build:
17+
runs-on: ubuntu-latest
18+
env:
19+
# This number is set "by eye": we want it to be big enough
20+
# so that it's bigger than the number of commits in any reasonable PR,
21+
# and also as small as possible since increasing the number makes
22+
# the initial `git fetch` slower.
23+
FETCH_DEPTH: 50
24+
strategy:
25+
matrix:
26+
# Only lint on the min and max supported Python versions.
27+
# It's extremely unlikely that there's a lint issue on any version in between
28+
# that doesn't show up on the min or max versions.
29+
#
30+
# GitHub rate-limits how many jobs can be running at any one time.
31+
# Starting new jobs is also relatively slow,
32+
# so linting on fewer versions makes CI faster.
33+
python-version:
34+
- "3.9"
35+
- "3.11"
36+
steps:
37+
- uses: actions/checkout@v3
38+
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
39+
uses: "./.github/actions/poetry_setup"
40+
with:
41+
python-version: ${{ matrix.python-version }}
42+
poetry-version: ${{ env.POETRY_VERSION }}
43+
working-directory: ${{ inputs.working-directory }}
44+
cache-key: lint-with-extras
45+
46+
- name: Check Poetry File
47+
shell: bash
48+
working-directory: ${{ inputs.working-directory }}
49+
run: |
50+
poetry check
51+
52+
- name: Check lock file
53+
shell: bash
54+
working-directory: ${{ inputs.working-directory }}
55+
run: |
56+
poetry lock --check
57+
58+
- name: Install dependencies
59+
# Also installs dev/lint/test/typing dependencies, to ensure we have
60+
# type hints for as many of our libraries as possible.
61+
# This helps catch errors that require dependencies to be spotted, for example:
62+
# https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
63+
#
64+
# If you change this configuration, make sure to change the `cache-key`
65+
# in the `poetry_setup` action above to stop using the old cache.
66+
# It doesn't matter how you change it, any change will cause a cache-bust.
67+
working-directory: ${{ inputs.working-directory }}
68+
run: |
69+
poetry install --with dev,lint,test,typing
70+
71+
- name: Get .mypy_cache to speed up mypy
72+
uses: actions/cache@v3
73+
env:
74+
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
75+
with:
76+
path: |
77+
${{ env.WORKDIR }}/.mypy_cache
78+
key: mypy-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', env.WORKDIR)) }}
79+
80+
- name: Analysing the code with our lint
81+
working-directory: ${{ inputs.working-directory }}
82+
run: |
83+
make lint

‎.github/workflows/_test.yml‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
name: test
2+
3+
on:
4+
workflow_call:
5+
inputs:
6+
working-directory:
7+
required: true
8+
type: string
9+
description: "From which folder this pipeline executes"
10+
11+
env:
12+
POETRY_VERSION: "1.7.1"
13+
14+
jobs:
15+
build:
16+
defaults:
17+
run:
18+
working-directory: ${{ inputs.working-directory }}
19+
runs-on: ubuntu-latest
20+
strategy:
21+
matrix:
22+
python-version:
23+
- "3.8"
24+
- "3.9"
25+
- "3.10"
26+
- "3.11"
27+
name: Python ${{ matrix.python-version }}
28+
steps:
29+
- uses: actions/checkout@v3
30+
31+
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
32+
uses: "./.github/actions/poetry_setup"
33+
with:
34+
python-version: ${{ matrix.python-version }}
35+
poetry-version: ${{ env.POETRY_VERSION }}
36+
working-directory: ${{ inputs.working-directory }}
37+
cache-key: core
38+
39+
- name: Install dependencies
40+
shell: bash
41+
run: poetry install
42+
43+
- name: Run core tests
44+
shell: bash
45+
run: make test
46+
47+
- name: Ensure the tests did not create any additional files
48+
shell: bash
49+
run: |
50+
set -eu
51+
52+
STATUS="$(git status)"
53+
echo "$STATUS"
54+
55+
# grep will exit non-zero if the target message isn't found,
56+
# and `set -e` above will cause the step to fail.
57+
echo "$STATUS" | grep 'nothing to commit, working tree clean'

‎.github/workflows/ci.yml‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
---
2+
name: Run CI Tests
3+
4+
on:
5+
push:
6+
branches: [ main ]
7+
pull_request:
8+
workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI
9+
10+
# If another push to the same PR or branch happens while this workflow is still running,
11+
# cancel the earlier run in favor of the next run.
12+
#
13+
# There's no point in testing an outdated version of the code. GitHub only allows
14+
# a limited number of job runners to be active at the same time, so it's better to cancel
15+
# pointless jobs early so that more useful jobs can run sooner.
16+
concurrency:
17+
group: ${{ github.workflow }}-${{ github.ref }}
18+
cancel-in-progress: true
19+
20+
env:
21+
POETRY_VERSION: "1.7.1"
22+
WORKDIR: "."
23+
24+
jobs:
25+
lint:
26+
uses:
27+
./.github/workflows/_lint.yml
28+
with:
29+
working-directory: .
30+
secrets: inherit
31+
test:
32+
timeout-minutes: 5
33+
runs-on: ubuntu-latest
34+
defaults:
35+
run:
36+
working-directory: ${{ env.WORKDIR }}
37+
services:
38+
postgres:
39+
# ensure postgres version this stays in sync with prod database
40+
# and with postgres version used in docker compose
41+
image: postgres:16
42+
env:
43+
# optional (defaults to `postgres`)
44+
POSTGRES_DB: langchain_test
45+
# required
46+
POSTGRES_PASSWORD: langchain
47+
# optional (defaults to `5432`)
48+
POSTGRES_PORT: 5432
49+
# optional (defaults to `postgres`)
50+
POSTGRES_USER: langchain
51+
ports:
52+
# maps tcp port 5432 on service container to the host
53+
- 5432:5432
54+
# set health checks to wait until postgres has started
55+
options: >-
56+
--health-cmd pg_isready
57+
--health-interval 3s
58+
--health-timeout 5s
59+
--health-retries 10
60+
strategy:
61+
matrix:
62+
python-version:
63+
- "3.8"
64+
- "3.9"
65+
- "3.10"
66+
- "3.11"
67+
name: Python ${{ matrix.python-version }} tests
68+
steps:
69+
- uses: actions/checkout@v3
70+
71+
- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
72+
uses: "./.github/actions/poetry_setup"
73+
with:
74+
python-version: ${{ matrix.python-version }}
75+
poetry-version: ${{ env.POETRY_VERSION }}
76+
working-directory: ${{ env.WORKDIR }}
77+
cache-key: langchain-extract-all
78+
- name: Test database connection
79+
run: |
80+
# Set up postgresql-client
81+
sudo apt-get install -y postgresql-client
82+
# Test psql connection
83+
psql -h localhost -p 5432 -U langchain -d langchain_test -c "SELECT 1;"
84+
env:
85+
# postgress password is required; alternatively, you can run:
86+
# `PGPASSWORD=postgres_password psql ...`
87+
PGPASSWORD: langchain
88+
89+
- name: Install dependencies
90+
shell: bash
91+
run: |
92+
echo "Running tests, installing dependencies with poetry..."
93+
poetry install --with test,lint,typing,docs
94+
95+
- name: Run tests
96+
run: make test
97+
98+
- name: Ensure the tests did not create any additional files
99+
shell: bash
100+
run: |
101+
set -eu
102+
103+
STATUS="$(git status)"
104+
echo "$STATUS"
105+
106+
# grep will exit non-zero if the target message isn't found,
107+
# and `set -e` above will cause the step to fail.
108+
echo "$STATUS" | grep 'nothing to commit, working tree clean'

0 commit comments

Comments
 (0)