Skip to content

Commit c06a209

Browse files
lsulaklsulak
authored
Merge pull request #119 from AbsaOSS/feature/118-db-function-cleanup-pre-118
#118: DB cleanup and IT tests addition to CI
2 parents 8ff2a10 + 950947b commit c06a209

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

45 files changed

+556
-526
lines changed

‎.github/workflows/build.yml‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,6 @@ jobs:
5151
with:
5252
path: |
5353
${{ github.workspace }}/core/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
54-
${{ github.workspace }}/examples/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
5554
${{ github.workspace }}/slick/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
5655
${{ github.workspace }}/doobie/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
5756
key: ${{ runner.os }}-${{ matrix.scalaShort }}-${{ hashFiles('**/jacoco.xml') }}
@@ -79,7 +78,6 @@ jobs:
7978
with:
8079
path: |
8180
${{ github.workspace }}/core/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
82-
${{ github.workspace }}/examples/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
8381
${{ github.workspace }}/slick/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
8482
${{ github.workspace }}/doobie/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
8583
key: ${{ runner.os }}-${{ matrix.scalaShort }}-${{ hashFiles('**/jacoco.xml') }}
@@ -95,7 +93,6 @@ jobs:
9593
${{ github.workspace }}/core/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml,
9694
${{ github.workspace }}/slick/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml,
9795
${{ github.workspace }}/doobie/target/scala-${{ matrix.scalaShort }}/jacoco/report/jacoco.xml
98-
# examples don't need code coverage - at least not now
9996
token: ${{ secrets.GITHUB_TOKEN }}
10097
min-coverage-overall: ${{ matrix.overall }}
10198
min-coverage-changed-files: ${{ matrix.changed }}

‎.github/workflows/integration_tests.yml‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,61 @@
1+
#
2+
# Copyright 2022 ABSA Group Limited
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
#
16+
17+
name: Run Integration Tests
18+
19+
on:
20+
pull_request:
21+
branches:
22+
- '**'
23+
types: [ opened, synchronize, reopened ]
24+
25+
jobs:
26+
run-it:
27+
runs-on: ubuntu-latest
28+
services:
29+
postgres:
30+
image: postgres:15
31+
env:
32+
POSTGRES_PASSWORD: postgres
33+
POSTGRES_DB: movies
34+
options: >-
35+
--health-cmd pg_isready
36+
--health-interval 10s
37+
--health-timeout 5s
38+
--health-retries 5
39+
ports:
40+
- 5432:5432
41+
strategy:
42+
matrix:
43+
scala: [ 2.12.17, 2.13.12 ]
44+
steps:
45+
- name: Checkout code
46+
uses: actions/checkout@v2
47+
- uses: coursier/cache-action@v5
48+
49+
- name: Setup Scala
50+
uses: olafurpg/setup-scala@v14
51+
with:
52+
java-version: "adopt@1.8"
53+
54+
- name: Prepare testing database
55+
run: sbt flywayMigrate
56+
57+
- name: Build and run IT tests for Doobie
58+
run: sbt "project faDBDoobie" ++${{matrix.scala}} it:test
59+
60+
- name: Build and run IT tests for Slick
61+
run: sbt "project faDBSlick" ++${{matrix.scala}} it:test

‎README.md‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -171,11 +171,11 @@ Code coverage will be generated on path:
171171

172172
### Integration tests
173173

174-
There are now integration tests as part of the project (at the time of writing they are in the _Slick_ module).
174+
There are now integration tests as part of the project (at the time of writing they are in
175+
the _Slick_ and _Doobie_ modules).
175176

176-
For the tests to work properly a running Postgres instance is needed. And then the following setup:
177-
* execute (content of) all `*.sql` files within `it/resources/sql/` folder within a posgres query tool
178-
* modify `it/resources/application.conf` to point to the database used in the previous point
177+
For the tests to work properly a running Postgres instance is needed as well as all DB objects must be placed on the DB.
178+
We automated this process, see `database/README.md` for more details.
179179

180180
How to execute the tests:
181181

@@ -185,4 +185,4 @@ sbt it:test
185185

186186
## How to Release
187187

188-
Please see [this file](RELEASE.md) for more details.
188+
Please see [this file](RELEASE.md) for more details.

‎build.sbt‎

Lines changed: 9 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,16 @@ lazy val commonJacocoReportSettings: JacocoReportSettings = JacocoReportSettings
4848
lazy val commonJacocoExcludes: Seq[String] = Seq(
4949
)
5050

51+
enablePlugins(FlywayPlugin)
52+
flywayUrl := FlywayConfiguration.flywayUrl
53+
flywayUser := FlywayConfiguration.flywayUser
54+
flywayPassword := FlywayConfiguration.flywayPassword
55+
flywayLocations := FlywayConfiguration.flywayLocations
56+
flywaySqlMigrationSuffixes := FlywayConfiguration.flywaySqlMigrationSuffixes
57+
libraryDependencies ++= flywayDependencies
58+
5159
lazy val parent = (project in file("."))
52-
.aggregate(faDbCore, faDBSlick, faDBDoobie, faDBExamples)
60+
.aggregate(faDbCore, faDBSlick, faDBDoobie)
5361
.settings(
5462
name := "root",
5563
libraryDependencies ++= rootDependencies(scalaVersion.value),
@@ -102,18 +110,4 @@ lazy val faDBDoobie = (project in file("doobie"))
102110
jacocoExcludes := commonJacocoExcludes
103111
)
104112

105-
lazy val faDBExamples = (project in file("examples"))
106-
.configs(IntegrationTest)
107-
.settings(
108-
name := "examples",
109-
libraryDependencies ++= examplesDependencies(scalaVersion.value),
110-
Test / parallelExecution := false,
111-
(Compile / compile) := ((Compile / compile) dependsOn printScalaVersion).value, // printScalaVersion is run with compile
112-
publish / skip := true
113-
).dependsOn(faDbCore, faDBSlick)
114-
.settings(
115-
jacocoReportSettings := commonJacocoReportSettings.withTitle(s"fa-db:examples Jacoco Report - scala:${scalaVersion.value}"),
116-
jacocoExcludes := commonJacocoExcludes
117-
)
118-
119113
sonatypeProfileName := "za.co.absa"

‎demo_database/README.md‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
## About
2+
3+
This module implements a simple database with many types of objects (tables, functions, data insertions, and more)
4+
that will be used in integration tests in these modules:
5+
* `doobie/src/it/`
6+
* `slick/src/it/`
7+
8+
## Deployment
9+
10+
How to set up database for local testing
11+
12+
### Using Docker
13+
14+
```zsh
15+
# start up postgres docker container (optional; instead you can create movies on your local postgres instance)
16+
docker run --name=movies -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=movies -p 5432:5432 -d postgres:16
17+
18+
# migrate scripts
19+
sbt flywayMigrate
20+
21+
# kill & remove docker container (optional; only if using dockerized postgres instance)
22+
docker kill aul_db
23+
docker rm aul_db
24+
```
25+
26+
### Using local postgres instance
27+
28+
```zsh
29+
# migrate scripts
30+
sbt flywayMigrate
31+
```
32+
33+
In case some structures are already present in the database, you can use
34+
```zsh
35+
sbt flywayClean
36+
```
37+
to remove them or
38+
```zsh
39+
sbt flywayBaseline
40+
```
41+
to set the current state as the baseline.

‎demo_database/src/main/postgres/00_databases.ddl‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
CREATE DATABASE movies
17+
WITH
18+
ENCODING = 'UTF8'
19+
CONNECTION LIMIT = -1;

‎demo_database/src/main/postgres/V1.1.1__01_add_extensions.ddl‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*
2+
* Copyright 2021 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*/
15+
16+
CREATE EXTENSION IF NOT EXISTS hstore;
17+
CREATE EXTENSION IF NOT EXISTS ltree;
18+
CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; -- for function `uuid_generate_v4`

‎demo_database/src/main/postgres/integration/V1.2.10__get_all_date_time_types.sql‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright 2022 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
CREATE OR REPLACE FUNCTION integration.get_all_date_time_types(p_id INT)
18+
RETURNS TABLE(
19+
offset_date_time TIMESTAMPTZ,
20+
instant TIMESTAMPTZ,
21+
zoned_date_time TIMESTAMPTZ,
22+
local_date_time TIMESTAMP,
23+
local_date DATE,
24+
local_time TIME,
25+
sql_date DATE,
26+
sql_time TIME,
27+
sql_timestamp TIMESTAMP,
28+
util_date TIMESTAMP
29+
) AS $$
30+
BEGIN
31+
RETURN QUERY SELECT
32+
T.offset_date_time,
33+
T.instant,
34+
T.zoned_date_time,
35+
T.local_date_time,
36+
T.local_date,
37+
T.local_time,
38+
T.sql_date,
39+
T.sql_time,
40+
T.sql_timestamp,
41+
T.util_date
42+
FROM integration.date_time_types T limit p_id;
43+
END;
44+
$$ LANGUAGE plpgsql;

‎demo_database/src/main/postgres/integration/V1.2.11__insert_dates_times.sql‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
/*
2+
* Copyright 2022 ABSA Group Limited
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
CREATE OR REPLACE FUNCTION integration.insert_dates_times(
18+
IN p_offset_date_time TIMESTAMPTZ,
19+
IN p_instant TIMESTAMPTZ,
20+
IN p_zoned_date_time TIMESTAMPTZ,
21+
IN p_local_date_time TIMESTAMP,
22+
IN p_local_date DATE,
23+
IN p_local_time TIME,
24+
IN p_sql_date DATE,
25+
IN p_sql_time TIME,
26+
IN p_sql_timestamp TIMESTAMP,
27+
IN p_util_date DATE,
28+
OUT status INTEGER,
29+
OUT status_text TEXT,
30+
OUT o_id INTEGER
31+
) RETURNS record AS $$
32+
BEGIN
33+
INSERT INTO integration.date_time_types (
34+
offset_date_time,
35+
instant,
36+
zoned_date_time,
37+
local_date_time,
38+
local_date,
39+
local_time,
40+
sql_date,
41+
sql_time,
42+
sql_timestamp,
43+
util_date
44+
) VALUES (
45+
p_offset_date_time,
46+
p_instant,
47+
p_zoned_date_time,
48+
p_local_date_time,
49+
p_local_date,
50+
p_local_time,
51+
p_sql_date,
52+
p_sql_time,
53+
p_sql_timestamp,
54+
p_util_date
55+
) RETURNING id INTO o_id;
56+
57+
status := 11;
58+
status_text := 'OK';
59+
60+
RETURN;
61+
END;
62+
$$ LANGUAGE plpgsql;

‎doobie/src/it/database/other_types.sql‎ renamed to ‎demo_database/src/main/postgres/integration/V1.2.12__insert_other_types.sql‎

Lines changed: 2 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -14,51 +14,7 @@
1414
* limitations under the License.
1515
*/
1616

17-
CREATE TABLE runs.other_types (
18-
id INT PRIMARY KEY,
19-
ltree_col LTREE,
20-
inet_col INET,
21-
macaddr_col MACADDR,
22-
hstore_col HSTORE,
23-
cidr_col CIDR,
24-
json_col JSON,
25-
jsonb_col JSONB,
26-
uuid_col UUID,
27-
array_col INT[]
28-
);
29-
30-
INSERT INTO runs.other_types VALUES (
31-
1,
32-
'Top.Science.Astronomy',
33-
'192.168.1.1',
34-
'08:00:2b:01:02:03',
35-
'key=>value',
36-
'192.168.1/24',
37-
'{"key": "value"}',
38-
'{"key": "value"}',
39-
uuid_generate_v4(),
40-
ARRAY[1,2,3]
41-
);
42-
43-
CREATE OR REPLACE FUNCTION runs.read_other_types(p_id INT)
44-
RETURNS TABLE(
45-
id INT,
46-
ltree_col LTREE,
47-
inet_col INET,
48-
macaddr_col MACADDR,
49-
hstore_col HSTORE,
50-
cidr_col CIDR,
51-
json_col JSON,
52-
jsonb_col JSONB,
53-
uuid_col UUID,
54-
array_col INT[]
55-
) AS $$
56-
BEGIN
57-
RETURN QUERY SELECT * FROM runs.other_types T WHERE T.id = p_id;
58-
END;
59-
$$ LANGUAGE plpgsql;
60-
61-
CREATE OR REPLACE FUNCTION runs.insert_other_types(
17+
CREATE OR REPLACE FUNCTION integration.insert_other_types(
6218
p_id INT,
6319
p_ltree_col LTREE,
6420
p_inet_col INET,
@@ -76,7 +32,7 @@ CREATE OR REPLACE FUNCTION runs.insert_other_types(
7632
) AS $$
7733
BEGIN
7834
BEGIN
79-
INSERT INTO runs.other_types VALUES (
35+
INSERT INTO integration.other_types VALUES (
8036
p_id,
8137
p_ltree_col,
8238
p_inet_col,

0 commit comments

Comments
 (0)