Skip to content

Commit 932eb48

Browse files
j-collj-coll
committed
Merge branch 'develop' into TASK-6217
2 parents ea31d4b + b579d0c commit 932eb48

File tree

9 files changed

+177
-7
lines changed

9 files changed

+177
-7
lines changed

‎.github/workflows/build-java-app-workflow.yml‎

Lines changed: 72 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,36 +10,103 @@ on:
1010
type: string
1111
required: false
1212
default: "build-folder"
13+
upload_artifact:
14+
type: boolean
15+
required: false
16+
default: true
17+
dependency_repos:
18+
# Comma-separated list of dependency repositories to clone and build before building the main project.
19+
type: string
20+
required: false
21+
default: ""
22+
needs_hadoop_preparation:
23+
type: boolean
24+
required: false
25+
default: false
26+
hadoop_flavour:
27+
type: string
28+
required: false
29+
default: ""
30+
java_commons_libs_branch:
31+
type: string
32+
required: false
33+
default: "develop"
1334
outputs:
1435
version:
1536
description: "Project version"
1637
value: ${{ jobs.build-workflow.outputs.version }}
38+
cache_key:
39+
description: "Cache key used for Maven repository"
40+
value: ${{ jobs.build-workflow.outputs.cache_key }}
1741

1842
jobs:
1943
build-workflow:
2044
name: Build Java app
2145
runs-on: ${{ vars.UBUNTU_VERSION }}
2246
outputs:
2347
version: ${{ steps.get_project_version.outputs.version }}
48+
cache_key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
2449
steps:
2550
- uses: actions/checkout@v4
51+
name: Checkout main repository
52+
id: "checkout-main"
53+
## This checkout pulls the code of the repository where this workflow is being used
2654
with:
2755
fetch-depth: '10'
56+
- uses: actions/checkout@v4
57+
if: ${{ inputs.dependency_repos != '' }}
58+
name: Checkout java-common-libs repository
59+
id: "checkout-java-common-libs"
60+
## This checkout pulls the code of the java-common-libs repository to get the scripts
61+
## Checkout to "java-common-libs" folder.
62+
## Filter by ".github" folder
63+
with:
64+
repository: opencb/java-common-libs
65+
ref: ${{ inputs.java_commons_libs_branch }}
66+
path: java-common-libs
67+
fetch-depth: '1'
2868
- name: Set up JDK 8
2969
uses: actions/setup-java@v4
3070
with:
3171
distribution: 'temurin'
3272
java-version: '8'
3373
cache: 'maven'
34-
- name: Install dependencies branches
74+
- name: Clone dependencies
75+
id: clone_dependencies
76+
if: ${{ inputs.dependency_repos != '' }}
77+
run: |
78+
if [ -f "java-common-libs/.github/workflows/scripts/get_same_branch.sh" ]; then
79+
chmod +x java-common-libs/.github/workflows/scripts/get_same_branch.sh
80+
export DEPENDENCIES_SHA=${{ github.sha }}
81+
java-common-libs/.github/workflows/scripts/get_same_branch.sh "${{ github.ref_name }}" "${{ inputs.dependency_repos }}"
82+
fi
83+
- name: Cache local Maven repository
84+
id: cache
85+
uses: actions/cache@v4
86+
with:
87+
path: ~/.m2/repository
88+
key: ${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-${{ steps.clone_dependencies.outputs.dependencies_sha }}
89+
restore-keys: |
90+
${{ runner.os }}-maven-${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}-
91+
- name: Compile dependencies
92+
if: ${{ inputs.dependency_repos != '' }}
3593
run: |
36-
if [ -f "./.github/workflows/scripts/get_same_branch.sh" ]; then
37-
chmod +x ./.github/workflows/scripts/get_same_branch.sh
38-
./.github/workflows/scripts/get_same_branch.sh ${{ github.ref_name }}
94+
if [ -f "java-common-libs/.github/workflows/scripts/compile_same_branch.sh" ]; then
95+
chmod +x java-common-libs/.github/workflows/scripts/compile_same_branch.sh
96+
java-common-libs/.github/workflows/scripts/compile_same_branch.sh "${{ inputs.dependency_repos }}"
3997
fi
98+
- name: Prepare Hadoop profile
99+
if: ${{ inputs.needs_hadoop_preparation == true }}
100+
run: |
101+
chmod +x ./.github/workflows/scripts/prepare_hadoop.sh
102+
./.github/workflows/scripts/prepare_hadoop.sh --hadoop-flavour "${{ inputs.hadoop_flavour || vars.HADOOP_FLAVOUR }}"
103+
env:
104+
THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
105+
40106
- name: Maven Build (skip tests)
41107
run: mvn -T 2 clean install -DskipTests ${{ inputs.maven_opts }} --no-transfer-progress
42108
- uses: actions/upload-artifact@v4
109+
if: ${{ inputs.upload_artifact == true }}
43110
with:
44111
name: ${{ inputs.build_folder }}
45112
path: build
@@ -49,3 +116,4 @@ jobs:
49116
echo "version=`mvn help:evaluate -q -Dexpression=project.version -DforceStdout`" >> $GITHUB_OUTPUT
50117
- name: test-version-from-check
51118
run: echo "Project version is " ${{ steps.get_project_version.outputs.version }}
119+

‎.github/workflows/deploy-maven-repository-workflow.yml‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,9 @@ on:
66
maven_opts:
77
type: string
88
required: false
9+
cache_key:
10+
type: string
11+
required: false
912
secrets:
1013
MAVEN_GPG_PASSPHRASE:
1114
required: true
@@ -32,6 +35,14 @@ jobs:
3235
server-password: MAVEN_NEXUS_PASSWORD # env variable for token in deploy
3336
gpg-private-key: ${{ secrets.MAVEN_GPG_PRIVATE_KEY }} # Value of the GPG private key to import
3437
gpg-passphrase: MAVEN_GPG_PASSPHRASE # env variable for GPG private key passphrase
38+
- name: Cache local Maven repository
39+
uses: actions/cache/restore@v4
40+
if: ${{ inputs.cache_key != '' }}
41+
with:
42+
path: ~/.m2/repository
43+
key: ${{ inputs.cache_key }}
44+
## Force cache hit to avoid analyzing with incomplete dependencies
45+
fail-on-cache-miss: true
3546
- name: Deploy to Maven Central repository
3647
run: mvn clean deploy -DskipTests -Pdeploy-maven ${{ inputs.maven_opts }} --no-transfer-progress
3748
env:

‎.github/workflows/develop.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,3 +14,5 @@ jobs:
1414
uses: ./.github/workflows/deploy-maven-repository-workflow.yml
1515
needs: build
1616
secrets: inherit
17+
with:
18+
cache_key: ${{ needs.build.outputs.cache_key }}

‎.github/workflows/release.yml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ jobs:
1313
uses: ./.github/workflows/deploy-maven-repository-workflow.yml
1414
needs: build
1515
secrets: inherit
16+
with:
17+
cache_key: ${{ needs.build.outputs.cache_key }}
1618

1719
release:
1820
uses: ./.github/workflows/release-github-workflow.yml

‎.github/workflows/scripts/compile_same_branch.sh‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
2+
WORKSPACE=${WORKSPACE:-/home/runner/work/}
3+
4+
5+
function compile() {
6+
local REPO=$1
7+
if [ ! -d "${WORKSPACE}/$REPO" ]; then
8+
echo "Directory ${WORKSPACE}/$REPO does not exist. Skip compile"
9+
return 0;
10+
fi
11+
echo "::group::Compiling '$REPO' project from branch $BRANCH_NAME"
12+
cd "${WORKSPACE}/$REPO" || exit 2
13+
mvn clean install -DskipTests --no-transfer-progress
14+
echo "::endgroup::"
15+
}
16+
17+
## Comma separated list of repos to compile
18+
REPOS=$1
19+
20+
IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
21+
for REPO in "${REPO_ARRAY[@]}"; do
22+
compile "$REPO"
23+
done

‎.github/workflows/scripts/get_same_branch.sh‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
#!/bin/bash
2+
3+
BRANCH_NAME=$1
4+
## Comma separated list of repos to checkout
5+
REPOS=$2
6+
DEPENDENCIES_SHA=${DEPENDENCIES_SHA:-""}
7+
WORKSPACE=${WORKSPACE:-/home/runner/work/}
8+
9+
if [[ -z "$BRANCH_NAME" ]]; then
10+
echo "The first parameter is mandatory and must be a valid branch name."
11+
exit 1
12+
fi
13+
14+
function checkout(){
15+
local REPO=$1
16+
echo "::group::Installing '$REPO' project from branch $BRANCH_NAME"
17+
cd "${WORKSPACE}" || exit 2
18+
git clone https://github.com/opencb/"$REPO".git -b "$BRANCH_NAME"
19+
if [ -d "./$REPO" ]; then
20+
cd "$REPO" || exit 2
21+
DEPENDENCIES_SHA=${DEPENDENCIES_SHA}:$(git rev-parse HEAD)
22+
echo "Branch name $BRANCH_NAME already exists."
23+
else
24+
echo "Branch name $BRANCH_NAME does not exist in $REPO repository. Skipping installation."
25+
fi
26+
echo "::endgroup::"
27+
}
28+
29+
IFS=',' read -ra REPO_ARRAY <<<"$REPOS"
30+
for REPO in "${REPO_ARRAY[@]}"; do
31+
checkout "$REPO"
32+
done
33+
34+
## Apply sha1 to DEPENDENCIES_SHA if contains `:`
35+
if [[ "$DEPENDENCIES_SHA" == *":"* ]]; then
36+
DEPENDENCIES_SHA=$(echo -n "$DEPENDENCIES_SHA" | sha1sum | awk '{print $1}')
37+
fi
38+
39+
## Export DEPENDENCIES_SHA as github output
40+
echo "dependencies_sha=$DEPENDENCIES_SHA" >> "$GITHUB_OUTPUT"

‎.github/workflows/test-analysis.yml‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,10 @@ name: Build and test the project
22

33
on:
44
workflow_call:
5+
inputs:
6+
cache_key:
7+
type: string
8+
required: false
59
secrets:
610
SONAR_TOKEN:
711
required: true
@@ -20,6 +24,14 @@ jobs:
2024
distribution: 'temurin'
2125
java-version: '11'
2226
cache: 'maven'
27+
- name: Cache local Maven repository
28+
uses: actions/cache/restore@v4
29+
if: ${{ inputs.cache_key != '' }}
30+
with:
31+
path: ~/.m2/repository
32+
key: ${{ inputs.cache_key }}
33+
## Force cache hit to avoid analyzing with incomplete dependencies
34+
fail-on-cache-miss: true
2335
- name: Start MongoDB v6.0
2436
uses: supercharge/mongodb-github-action@1.8.0
2537
with:

‎.github/workflows/test-xetabase-workflow.yml‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -86,21 +86,33 @@ jobs:
8686
env:
8787
SSH_HOST: ${{ env.SSH_HOST }}
8888
SSH_PORT: ${{ env.SSH_PORT }}
89+
- name: Cache local Maven repository
90+
id: cache
91+
uses: actions/cache@v4
92+
with:
93+
path: ~/.m2/repository
94+
key: ${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-${{ github.sha }}
95+
restore-keys: |
96+
${{ runner.os }}-maven-${{ vars.HADOOP_FLAVOUR }}-
8997
- name: Run all OpenCB Junit tests, ie. java-common-libs, biodata, opencga and opencga-enterprise
9098
run: |
9199
cd opencga-enterprise
92100
ln -s ../opencga opencga-home
93-
./build.sh -t -l runShortTests -b -s -T ${{ inputs.task }}
101+
./build.sh --test --test-level runShortTests --prepare-branches --test-save-reports --task ${{ inputs.task }} --storage-hadoop ${{ vars.HADOOP_FLAVOUR }} --prepare-hadoop
102+
env:
103+
THIRDPARTY_READ_TOKEN: ${{ secrets.THIRDPARTY_READ_TOKEN }}
94104
- name: Upload reports results to Github
95105
uses: actions/upload-artifact@v4
96106
with:
97107
name: report-test
98-
path: ./opencga-enterprise/reports/test
108+
path: ./opencga-enterprise/reports/
109+
if-no-files-found: error
99110
- name: Upload log
100111
uses: actions/upload-artifact@v4
101112
with:
102113
name: build-log
103114
path: ./opencga-enterprise/build.log
115+
if-no-files-found: error
104116
- name: Log summary
105117
run: |
106118
cat ./opencga-enterprise/build.log | tee -a $GITHUB_STEP_SUMMARY

‎pom.xml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
<jackson.version>2.14.3</jackson.version>
2323
<apache.commons.lang3.version>3.14.0</apache.commons.lang3.version>
2424
<slf4j.version>1.7.36</slf4j.version>
25-
<avro.version>1.7.7</avro.version>
25+
<avro.version>1.11.4</avro.version>
2626
<mongodb.version>4.11.5</mongodb.version>
2727
<solr.version>8.11.3</solr.version>
2828
<jcommander.version>1.69</jcommander.version>

0 commit comments

Comments
 (0)