Sqlite and MySql support for asynk (#141)

* possible impl with sqlx, we need to refactor dependencies and features and also probably need to make queries folder for `sqlite` and `mysql` * fix clippy xd * finally sqlite almost work, need help to debug something * deleting some expects * sqlite may work in workflow * stupid workflow work * plz work * plz work , think i have fixed * fix workflow , now will work :D * this should work * this will definitely work * clean up so it works incorrectly faster * okay that didn't work so idk what i'm doing here * if this works i'm dropping out of university * store uuid as text , so this way `PostgreSQL` schema will not change * fix comment * fix clippy * implement backend sqlx trait that allows better code * better implementation with enum variants * `Makefile` now creates `tests_sqlite` directory * make `asynk` tests use `.env` variables * drop unnecessary `drop` * query types as enum * fix clippy * fix fmt, my bad i had a missconf in my home computer xd * stupid mysql does not work anything , i hate it so much :/ * rework to make it work with MySQL * debugging issue : the issue is related to how a uniq task is inserted in MySQL backend * MySQL own mutex , why Mysql is so gae ? * fix clippy * split test jobs * fix build * fix github actions * do not run github actions twice * fix formatting build * another approach * tokio console debug * fixing workflow * fixing workflow again * i think i finally fixed * fixed \! * fixed \? * i am stupid now should be fixed :) * test blocking and fang derive error * add postgres service to blocking workflow * running async worker tests and blocking worker tests * cargo term color sqlite * change sqlx version to 0.6.3 and using Pool instead of Transactions * fix workflow for mysql * fix workflow * fix clippy, warns and mysql workflow * adress comments and fix workflow * fix mysql healthcheck * fix mysql healthcheck, yeap again * remove healthcheck * healthcheck is needed , plz work * plz just work :( * try to fix workflow * update workflow to use a non deprecated version * return an error instead of panicking with fock * deleting prints and updating comments * deleting unwraps or justifying them * README and moving each backend to separate files * final README * bump fang version and bump rust version * nit: added missing newline for consistency * update readme * bump fang version following semver * address ayrat issues * Fix/independent decoding (#149) * independent decoding * Postgres and Sqlite passing * fix unreachable pattern warn * delete vscode stuff * I think this may be a Rust compiler issue, but I fixed it so I will take it * fix clippy * relocate comment * remove `backend` field from `AsyncQueue` * fix clippy * worst clippy fix of all time * fmt fix * use variant NoBackend instead Option * fix clippy * delete dummy variant * changing sqlite and mysql migrations * sqlx uuid encoding and decoding for sqlite and postgres * delete uuid previous impl comment * fix retry_task impl * fix mysql timestamp precision * rework encoding Datime<Utc> for postgresql and sqlite * rework encoding Uuid for mysql * fix clippy * fix serde_json::Value decoding and encoding * README stuff * improving API * improving worker API --------- Co-authored-by: Dopplerian <dopplerian+git@proton.me> Co-authored-by: Ayrat Badykov <ayratin555@gmail.com>
ayrat555 · Apr 19, 2024 · bcbb48d · bcbb48d
1 parent 4bb21c4
commit bcbb48d
Show file tree

Hide file tree

Showing 76 changed files with 2,263 additions and 702 deletions.
diff --git a/.env b/.env
@@ -20,8 +20,11 @@ SQLITE_FILE=fang.db
 SQLITE_DIESEL_DIR=fang/sqlite_migrations
 SQLITE_MIGRATIONS=${SQLITE_DIESEL_DIR}/migrations
 SQLITE_CONFIG=${SQLITE_DIESEL_DIR}/diesel.toml
+SQLITE_TESTS_DIR=tests_sqlite
 
 HOST=127.0.0.1
-POSTGRES_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${HOST}/${POSTGRES_DB}
-MYSQL_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@${HOST}/${MYSQL_DB}
+POSTGRES_BASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${HOST}
+POSTGRES_URL=${POSTGRES_BASE_URL}/${POSTGRES_DB}
+MYSQL_BASE_URL=mysql://${MYSQL_USER}:${MYSQL_PASSWORD}@${HOST}
+MYSQL_URL=${MYSQL_BASE_URL}/${MYSQL_DB}
 DATABASE_URL=${POSTGRES_URL}
diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml
@@ -3,17 +3,40 @@ name: Test and Build Rust
 on:
   push:
   pull_request:
+    types: [opened, reopened]
   schedule:
     # Check if it works with current dependencies (weekly on Wednesday 2:32 UTC)
     - cron: '32 2 * * 3'
 
-env :
-  DATABASE_URL : postgres://postgres:postgres@localhost/fang
 
 jobs:
-  test:
-    name: Test
+  clippy:
+    name: Clippy
     runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
+
+    strategy:
+      matrix:
+        toolchain:
+          - stable
+
+    steps:
+      - uses: actions/checkout@v4
+
+
+      - name: Setup Rust
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
+
+
+      - name: Run clippy
+        run: cargo clippy --verbose --all-targets --all-features -- -D warnings
+
+  test_postgres_blocking:
+    name: Test blocking
+    runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
 
     services:
       # Label used to access the service container
@@ -35,67 +58,196 @@ jobs:
           --health-timeout 5s
           --health-retries 5
 
+    strategy:
+      matrix:
+        toolchain:
+          - stable
+
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
       - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          components: clippy
-          override: true
-          profile: minimal
-          toolchain: stable
-
-      - name: Run clippy
-        uses: actions-rs/cargo@v1
-        with:
-          command: clippy
-          args: --verbose --all-targets --all-features -- -D warnings
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
 
       - name: Install diesel-cli
-        uses: actions-rs/cargo@v1
-        with:
-          command: install
-          args: diesel_cli --no-default-features --features "postgres"
+        run: cargo install diesel_cli --no-default-features --features postgres
 
       - name: Setup Postgres db
         working-directory: ./fang/postgres_migrations
-        run: diesel setup
+        run: diesel setup --database-url "postgres://postgres:postgres@localhost/fang"
+
+      - name: Run blocking tests
+        run: cargo test "blocking::queue::postgres" --verbose --features blocking --color always -- --nocapture
+
+      - name: Run blocking dirty tests
+        run: cargo test "blocking::worker" --verbose --features blocking -- --ignored
+
+  test_fang_derive_error:
+    name: Test fang_derive_error
+    runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
+
+    strategy:
+      matrix:
+        toolchain:
+          - stable
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
+
+      - name: Run fang derive error tests
+        run: cargo test "fang_derive_error" --verbose --color always -- --nocapture
+
+  test_postgres:
+    name: Test postgres
+    runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: postgres://postgres:postgres@localhost/fang
+      CARGO_TERM_COLOR: always
+
+    strategy:
+      matrix:
+        toolchain:
+          - stable
+
+    services:
+      # Label used to access the service container
+      postgres:
+        # Docker Hub image
+        image: postgres
+        # Provide the password for postgres
+        env:
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_USER: postgres
+        # Set health checks to wait until postgres has started
+
+        ports:
+          - 5432:5432
+
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
 
-      - name: Change working dir
-        working-directory: ./../..
-        run: pwd
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
+
+      - name: Install diesel-cli
+        run: cargo install diesel_cli --no-default-features --features postgres
+
+      - name: Setup Postgres db
+        working-directory: ./fang/postgres_migrations
+        run: diesel setup --database-url "postgres://postgres:postgres@localhost/fang"
 
       - name: Run tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --verbose --all-features
+        run: cargo test "asynk::async_queue::postgres" --verbose --features asynk-postgres --color always -- --nocapture
+
+      - name: Run worker tests
+        run: cargo test "asynk::async_worker::async_worker_tests" --verbose --features asynk-postgres --color always -- --nocapture
+
+  test_sqlite:
+    name: Test sqlite
+    runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
+
+    strategy:
+      matrix:
+        toolchain:
+          - stable
 
-      - name: Run dirty tests
-        uses: actions-rs/cargo@v1
-        with:
-          command: test
-          args: --verbose --all-features -- --ignored
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
+
+      - name: Install sqlite3
+        run: |
+          sudo apt install -y sqlite3
+          sqlite3 fang.db "VACUUM;"
+          mkdir tests_sqlite
+
+      - name: Install diesel-cli
+        run: cargo install diesel_cli --no-default-features --features sqlite
+
+      - name: Setup Sqlite db
+        working-directory: ./fang/sqlite_migrations
+        run: diesel setup --database-url "sqlite3://../../../fang.db"
+
+      - name: Run tests
+        run: cargo test "asynk::async_queue::sqlite" --verbose --features asynk-sqlite -- --nocapture
 
   release:
     name: Release x86_64-unknown-linux-gnu
     runs-on: ubuntu-latest
-    needs: test
+    env:
+      CARGO_TERM_COLOR: always
 
-    steps:
-      - uses: actions/checkout@v3
+    strategy:
+      matrix:
+        toolchain:
+          - stable
 
+    steps:
+      - uses: actions/checkout@v4
       - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          override: true
-          profile: minimal
-          target: x86_64-unknown-linux-gnu
-          toolchain: stable
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
 
       - name: Build release
-        uses: actions-rs/cargo@v1
-        with:
-          command: build
-          args: --release --verbose --all-features --target x86_64-unknown-linux-gnu
+        run: cargo build --release --verbose --all-features --target x86_64-unknown-linux-gnu
+
+  test_mysql:
+    name: Test mysql
+    runs-on: ubuntu-latest
+    env:
+      DATABASE_URL: mysql://root:mysql@localhost/fang
+      CARGO_TERM_COLOR: always
+
+    strategy:
+      matrix:
+        toolchain:
+          - stable
+
+    services:
+      # Label used to access the service container
+      mysql:
+        # Docker Hub image
+        image: mysql:8.1
+        # Provide the password for postgres
+        env:
+          MYSQL_ROOT_PASSWORD: mysql
+          MYSQL_DATABASE: fang
+        # here we should check if mysql is ready, but this does not work
+        options: >-
+          --health-cmd "mysqladmin ping -h localhost -u root -pmysql"
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+
+        ports:
+          - 3306:3306
+
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Setup Rust
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
+
+      - name: Install diesel-cli
+        run: cargo install diesel_cli --no-default-features --features mysql
+
+      - name: Setup MySQL db
+        working-directory: ./fang/mysql_migrations
+        run: diesel setup --database-url "mysql://root:mysql@127.0.0.1/fang"
+
+      - name: Run tests
+        run: cargo test "asynk::async_queue::mysql" --verbose --features asynk-mysql --color always -- --nocapture
diff --git a/.github/workflows/style.yml b/.github/workflows/style.yml
@@ -3,24 +3,23 @@ name: Rust Code Formatting
 on:
   push:
   pull_request:
+    types: [opened, reopened]
 
 jobs:
   rustfmt:
     name: Rustfmt
     runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
+    strategy:
+      matrix:
+        toolchain:
+          - stable
     steps:
       - uses: actions/checkout@v3
 
       - name: Setup Rust
-        uses: actions-rs/toolchain@v1
-        with:
-          toolchain: stable
-          override: true
-          profile: minimal
-          components: rustfmt
+        run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }}
 
       - name: Check format
-        uses: actions-rs/cargo@v1
-        with:
-          command: fmt
-          args: -- --check --verbose
+        run: cargo fmt -- --check --verbose
diff --git a/.gitignore b/.gitignore
@@ -3,3 +3,4 @@ Cargo.lock
 docs/content/docs/CHANGELOG.md
 docs/content/docs/README.md
 fang.db
+tests_sqlite/
diff --git a/Makefile b/Makefile
@@ -47,6 +47,7 @@ db_mysql:
 db_sqlite:
 	@echo -e $(BOLD)Setting up SQLite database...$(END_BOLD)
 	sqlite3 "$(SQLITE_FILE)" "VACUUM;"
+	mkdir -p "$(SQLITE_TESTS_DIR)"
 	$(MAKE) diesel_sqlite
 
 ensure_diesel:

diff --git a/fang-derive-error/example/src/lib.rs b/fang-derive-error/example/src/lib.rs
@@ -7,7 +7,7 @@ pub enum MyAwesomeError {
     MyVariantErrorTwo(u32),
 }
 #[cfg(test)]
-mod tests {
+mod fang_derive_error_tests {
     use crate::MyAwesomeError;
     use fang::FangError;