Dev (#105)

* test files that skip race

* fix typos

* page object

* spec

* traefik label uniqueness

* actually use configurable api address

* cannot install playwright lib #microsoft/playwright/issues/23228

* fix tty check

* construct URL rest test util

* use cfg

* refactor

* tx private error

* remove useless address arg

* error refactoring

* validate response refactor - default only in tests

* pg errors

* notes on descriptive xo errors for end users

* contextual errors in xo

* better rest errors

* private error hidden

* update

* include types in error res

* traefik update and move certificate creation to traefik setup

* update

* fix typos

* whitespace

* frontend api path utils

* notes on excluded index generation

* refactor and add caching to test watch default

* allow tag selection for keys extraction

* generics notes

* proper skipping of excluded index types

* notes for required xo generic query function

* broken - TODO adapt xo index gen for base filter function

* broken - working on params fix

* fix duplicate base query gen and remove old user nonunique index

* add xo tests description

* add trigram indexes to xo test schema - no tests yet

* test trigram custom filters

* test trigram filtering

* ilike parameter escape workaround

.github/workflows/tests.yaml

@@ -50,6 +50,8 @@ jobs:
               project test.backend -race -count=40 --x-env=ci
           done
 
+          echo "Testing files with !race directive"
+          project test.backend -count=40 --x-env=ci
 
       - name: Test frontend
         run: |

.vscode/launch.json

@@ -18,7 +18,9 @@
       "mode": "auto",
       "cwd": ".",
       "program": "cmd/rest-server/main.go",
-      "args": ["-env=.env.dev"]
+      "args": [
+        "-env=.env.dev"
+      ]
     },
     {
       "name": "Launch CLI",
@@ -27,7 +29,9 @@
       "mode": "auto",
       "cwd": ".",
       "program": "cmd/cli/main.go",
-      "args": ["-env=.env.dev"]
+      "args": [
+        "-env=.env.dev"
+      ]
     }
   ]
 }

bin/.helpers.sh

@@ -48,6 +48,7 @@ ensure_pwd_is_top_level() {
 # Prompt the user for confirmation.
 # Most likely will want to always run as `with_tty confirm ...`
 confirm() {
+  test -n "$CI" && return
   test -n "$NO_CONFIRMATION" && return
 
   local prompt="$1"

bin/.project.dependencies.sh

@@ -256,10 +256,6 @@ install.bin.mkcert() {
     wget https://github.com/FiloSottile/mkcert/releases/download/v"$VERSION"/mkcert-v"$VERSION"-linux-amd64 -O mkcert
     chmod +x mkcert
     mv mkcert ./bin/tools/
-    cd "$CERTIFICATES_DIR" || exit
-    echo "Setting up local certificates"
-    mkcert --cert-file localhost.pem --key-file localhost-key.pem "localhost" "*.e2e.localhost" "*.local.localhost" "*.dev.localhost" "*.ci.localhost" "*.prod.localhost" "127.0.0.1" "::1" "host.docker.internal"
-    cd ..
     mkcert -install
   } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
 }

bin/project

@@ -132,6 +132,7 @@ x.install-tools() {
     go install github.com/danicc097/xo@v1.0.0 &
     go install github.com/mikefarah/yq/v4@v4.27.2 &
     go install github.com/hexdigest/gowrap/cmd/gowrap@latest &
+    go install golang.org/x/tools/cmd/stringer@latest &
 
     go install google.golang.org/protobuf/cmd/protoc-gen-go@v1.28.1 &
     go install github.com/planetscale/vtprotobuf/cmd/protoc-gen-go-vtproto@v0.2.0 &
@@ -168,7 +169,7 @@ x.gen.pregen() {
     missing_opids="$(yq e "(.paths[][] | select(has(\"operationId\") | not) | path) | join(\".\")" $SPEC)"
     [[ -n "$missing_opids" ]] && err "Paths missing operationId: $(printf "\t\n%s" ${missing_opids[*]})"
 
-    # spec must always be valid before generating. Schemas to generate must already exist with the appropiate vendor extension
+    # spec must always be valid before generating. Schemas to generate must already exist with the appropriate vendor extension
     pregen -env=".env.$env" --validate-spec-only
 
     sync_db_enums_with_spec
@@ -882,7 +883,7 @@ x.test.backend.watch() {
       while read -r event_time event_file 2>/dev/null || sleep $latency; do
         clear
 
-        { APP_ENV="$env" go test -tags skipxo -count=1 "$@" ./...; } && echo "${GREEN}✓ All tests passing${OFF}"
+        { APP_ENV="$env" go test -tags skipxo "$@" ./...; } && echo "${GREEN}✓ All tests passing${OFF}"
       done
   done
 }
@@ -993,7 +994,7 @@ x.recreate-shared-services() {
 x.release() {
   { { {
     search_stopship "STOPSHIP" &
-    # go mod verify & # TODO go.work issues solved in go 1.21 (https://github.com/golang/go/issues/54372)
+    GOWORK=off go mod verify & # (https://github.com/golang/go/issues/54372)
 
     wait_without_error
   } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
@@ -1007,13 +1008,36 @@ x.dev-utils.api-keys() {
 # Setups a traefik container with predefined configuration in `install-dir`.
 # Args: install-dir
 x.setup.traefik() {
-  git clone --depth=1 https://github.com/danicc097/traefik-bootstrap.git "$1"
-  docker network create traefik-net || true
-  mkdir -p "$1"/traefik/certificates
-  cp $CERTIFICATES_DIR/* "$1"/traefik/certificates
-  cd "$1" || exit
-  ./compose-up
-  cd - >/dev/null || exit
+  { { {
+    test -z "$1" && err "installation directory is required"
+
+    x.setup.mkcert
+
+    git clone --depth=1 https://github.com/danicc097/traefik-bootstrap.git "$1"
+    docker network create traefik-net || true
+    mkdir -p "$1"/traefik/certificates
+    cp $CERTIFICATES_DIR/* "$1"/traefik/certificates
+    cd "$1" || exit
+    cp traefik/dynamic_conf.yaml.example traefik/dynamic_conf.yaml
+    echo "Adding $PWD/certificates/"
+    yq e ".tls.certificates += [{
+    \"certFile\": \"$PWD/$CERTIFICATES_DIR/localhost.pem\",
+    \"keyFile\": \"$PWD/$CERTIFICATES_DIR/localhost-key.pem\"
+  }]" -i traefik/dynamic_conf.yaml
+
+    ./compose-up
+    cd - >/dev/null || exit
+  } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
+}
+
+# Installs mkcert local development certificates.
+x.setup.mkcert() {
+  { { {
+    cd "$CERTIFICATES_DIR" || exit
+    echo "Setting up local certificates"
+    mkcert --cert-file localhost.pem --key-file localhost-key.pem "localhost" "*.e2e.localhost" "*.local.localhost" "*.dev.localhost" "*.ci.localhost" "*.prod.localhost" "127.0.0.1" "::1" "host.docker.internal" 2>&1
+    cd -
+  } 2>&4 | xlog >&3; } 4>&1 | xerr >&3; } 3>&1
 }
 
 ########################## migrations ##########################
@@ -1119,7 +1143,6 @@ x.db.gen.initial-data() {
 
 # Seed database.
 x.db.initial-data() {
-  # xlog eats up read -p (prompt)
   { { {
     x.db.drop
     x.migrate up
@@ -1152,7 +1175,6 @@ x.db.dump() {
 # Restore the database with the latest dump or `file` for the current environment.
 # Args: [file]
 x.db.restore() {
-  # xlog eats up read -p (prompt)
   dump_file="$1"
   if [[ -n $dump_file ]]; then
     [[ ! -f $dump_file ]] && err "$dump_file does not exist"
@@ -1186,12 +1208,14 @@ x.e2e.run() {
   { { {
     name="$PROJECT_PREFIX-e2e"
     cd e2e
-    pnpm install -r --frozen-lockfile
+    pnpm i # cannot install playwright lib
     DOCKER_BUILDKIT=1 BUILDKIT_PROGRESS=plain docker build -t "$name" .
     cd - >/dev/null
 
-    # need symlink resolution
-    docker run -i --rm \
+    # need symlink resolution for data
+
+    test -t 0 && opts="-t"
+    docker run -i $opts --rm \
       --ipc=host \
       --network host \
       -v "$(pwd)/cmd/oidc-server/data/:/cmd/oidc-server/data/" \

cmd/rest-server/main.go

@@ -6,6 +6,7 @@ import (
 	"log"
 	"os/exec"
 	"runtime"
+	"strings"
 	"time"
 
 	"github.com/danicc097/openapi-go-gin-postgres-sqlc/internal"
@@ -35,15 +36,24 @@ func openBrowser(url string) {
 }
 
 func main() {
-	var env, address, specPath, scopePolicyPath, rolePolicyPath string
+	var env, specPath, scopePolicyPath, rolePolicyPath string
 
-	flag.StringVar(&env, "env", ".env", "Environment Variables filename")
-	flag.StringVar(&address, "address", ":8090", "HTTP Server Address")
+	flag.StringVar(&env, "env", "", "Environment Variables filename")
 	flag.StringVar(&specPath, "spec-path", "openapi.yaml", "OpenAPI specification filepath")
 	flag.StringVar(&rolePolicyPath, "roles-path", "roles.json", "Roles policy JSON filepath")
 	flag.StringVar(&scopePolicyPath, "scopes-path", "scopes.json", "Scopes policy JSON filepath")
 	flag.Parse()
 
+	var errs []string
+
+	if env == "" {
+		errs = append(errs, "    - env is required but unset")
+	}
+
+	if len(errs) > 0 {
+		log.Fatalf("error: \n" + strings.Join(errs, "\n"))
+	}
+
 	// go openBrowser(url)
 
 	// dummy values for dashboard
@@ -54,7 +64,7 @@ func main() {
 	prometheus.MustRegister(cpuTemp)
 	cpuTemp.Set(65.3)
 
-	errC, err := server.Run(env, address, specPath, rolePolicyPath, scopePolicyPath)
+	errC, err := server.Run(env, specPath, rolePolicyPath, scopePolicyPath)
 	if err != nil {
 		log.Fatalf("Couldn't run: %s", err)
 	}

db/migrations/0000002_init.up.sql

@@ -9,6 +9,8 @@ create extension if not exists pg_trgm schema extensions;
 
 create extension if not exists btree_gin schema extensions;
 
+create extension if not exists rum schema extensions;
+
 -- internal use. update whenever a project with its related workitems,
 --  etc. tables are created in migrations
 create table projects (
@@ -107,8 +109,6 @@ comment on column user_api_keys.user_api_key_id is '"properties":private';
 --   external_id is null;
 -- composite on id, deleted_at, email, deleted_at, etc. will not improve speed
 -- create unique index on users (user_id) where deleted_at is null; -- helps if you have much more deleted rows only
-create index on users (created_at);
-
 -- create index on users (deleted_at);  - not worth the extra overhead.
 -- for finding all deleted users exclusively
 create index on users (deleted_at)
@@ -317,6 +317,14 @@ create table work_items (
 
 create index on work_items (team_id);
 
+create index on work_items using gin (title gin_trgm_ops);
+
+create index on work_items using gin (description gin_trgm_ops);
+
+create index on work_items using gin (title gin_trgm_ops , description gin_trgm_ops);
+
+create index on work_items using gin (title , description gin_trgm_ops);
+
 
 /*
 when a new project is required -> manual table creation with empty new fields, just

docker-compose.yml

@@ -8,14 +8,14 @@ x-logging: &default-logging
 # Debug config with docker-compose --env-file <...> config
 services:
   backend:
-    image: ${PROJECT_PREFIX:?not set}-${APP_ENV}-backend:latest
+    image: ${PROJECT_PREFIX:?not set}-${APP_ENV:?not set}-backend:latest
     container_name: backend_${PROJECT_PREFIX}_${APP_ENV}
     networks:
       - traefik-net
     command:
       - "--env=.env.${APP_ENV}"
     volumes:
-      - ./.env.${APP_ENV}:/.env.${APP_ENV}
+      - ${PWD}/.env.${APP_ENV}:/.env.${APP_ENV}:ro
       # required since mock server will use these
       - /etc/ssl/certs/ca-certificates.crt:/etc/ssl/certs/ca-certificates.crt:ro
     build:

docker/docker-compose.oidc.yml

@@ -27,13 +27,13 @@ services:
       PORT: ${MOCK_OIDC_SERVER_PORT:?not set}
     labels:
       - traefik.enable=true
-      - traefik.http.routers.simple-oidc-server.rule=Host(`${OIDC_DOMAIN:?not set}`) && PathPrefix("/oidc")
-      - traefik.http.routers.simple-oidc-server.middlewares=strip_prefix_oidc # check docs to exclude requests from compression
-      - traefik.http.middlewares.strip_prefix_oidc.stripprefix.prefixes=/oidc
+      - traefik.http.routers.${PROJECT_PREFIX}-simple-oidc-server.rule=Host(`${OIDC_DOMAIN:?not set}`) && PathPrefix("/oidc")
+      - traefik.http.routers.${PROJECT_PREFIX}-simple-oidc-server.middlewares=${PROJECT_PREFIX}-simple-oidc-server-strip-prefix # check docs to exclude requests from compression
+      - traefik.http.middlewares.${PROJECT_PREFIX}-simple-oidc-server-strip-prefix.stripprefix.prefixes=/oidc
       - traefik.docker.network=traefik-net
-      - traefik.http.routers.simple-oidc-server.entrypoints=websecure
-      - traefik.http.routers.simple-oidc-server.tls=true
-      - traefik.http.services.simple-oidc-server.loadbalancer.server.port=${MOCK_OIDC_SERVER_PORT:?not set}
+      - traefik.http.routers.${PROJECT_PREFIX}-simple-oidc-server.entrypoints=websecure
+      - traefik.http.routers.${PROJECT_PREFIX}-simple-oidc-server.tls=true
+      - traefik.http.services.${PROJECT_PREFIX}-simple-oidc-server.loadbalancer.server.port=${MOCK_OIDC_SERVER_PORT:?not set}
     restart: unless-stopped
     logging: *default-logging
     healthcheck:

e2e/Dockerfile

@@ -4,3 +4,6 @@ WORKDIR /e2e
 
 ENV PATH /e2e/node_modules/.bin:$PATH
 
+
+
+

e2e/__tests__/example.spec.ts

@@ -1,5 +1,5 @@
 import { test, expect } from '@playwright/test'
-import authServerUsers from '../auth-server-users-e2e.json'
+import authServerUsers from '@users'
 
 test('users loaded', async ({ page }) => {
   expect(authServerUsers).toBeDefined()

e2e/__tests__/pages/login.ts

@@ -0,0 +1,29 @@
+import type { Page, BrowserContext } from 'playwright'
+
+export class LoginPage {
+  private readonly page: Page
+
+  constructor(page: Page) {
+    this.page = page
+  }
+
+  static async create(page: Page): Promise<LoginPage> {
+    const loginPage = new LoginPage(page)
+    await loginPage.navigateToLoginPage()
+    return loginPage
+  }
+
+  private async navigateToLoginPage(): Promise<void> {
+    await this.page.goto('<login_page_url>')
+  }
+
+  async enterCredentials(username: string, password: string): Promise<void> {
+    await this.page.fill('#username', username)
+    await this.page.fill('#password', password)
+  }
+
+  async submitLoginForm(): Promise<void> {
+    await this.page.click('button[type="submit"]')
+    await this.page.waitForURL('**/<target-path>')
+  }
+}

e2e/__tests__/specs/oidc-login.spec.ts

@@ -0,0 +1,13 @@
+import test, { expect } from '@playwright/test'
+import { LoginPage } from '__tests__/pages/login'
+import authServerUsers from '@users'
+
+test('Login redirects to auth server and back if authenticated', async ({ page }) => {
+  test.skip() // TODO:
+  const loginPage = await LoginPage.create(page)
+  const user1 = authServerUsers.user1
+  await loginPage.enterCredentials(user1.username, user1.password)
+  await loginPage.submitLoginForm()
+
+  expect(page.url()).toBe('<expected_redirect_url>')
+})

e2e/package.json

@@ -7,6 +7,9 @@
   "keywords": [],
   "author": "",
   "devDependencies": {
+    "typescript": "^4.8.2"
+  },
+  "dependencies": {
     "@playwright/test": "^1.35.0"
   }
 }