Skip to content

Commit

Permalink
Add SolidBench performance benchmark to CI
Browse files Browse the repository at this point in the history
  • Loading branch information
rubensworks committed Aug 29, 2024
1 parent 4aa2cf1 commit 6da1c4a
Show file tree
Hide file tree
Showing 52 changed files with 10,042 additions and 1 deletion.
1 change: 1 addition & 0 deletions .github/typedoc-json.sh
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ typedocjson='{
"extends": [
"../../typedoc.base.json"
],
"skipErrorChecking": true,
"entryPoints": [
"lib/index.ts"
]
Expand Down
89 changes: 89 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,95 @@ jobs:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

performance:
runs-on: ubuntu-latest
strategy:
matrix:
benchmark:
- solidbench
needs:
- test
- lint
if: startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/feature') || startsWith(github.ref, 'refs/heads/next')
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Use Node.js
uses: actions/setup-node@v4
with:
node-version: 18.x
- name: Install dependencies
run: yarn install --frozen-lockfile --ignore-engines
- name: Run benchmarks
run: cd performance/benchmark-${{ matrix.benchmark }}/ && yarn run performance:ci
- uses: actions/upload-artifact@v4
with:
name: performance-benchmark-${{ matrix.benchmark }}
path: performance/benchmark-${{ matrix.benchmark }}/combinations/combination_0/output/
- name: Print raw output
run: cat performance/benchmark-${{ matrix.benchmark }}/combinations/combination_0/output/query-times.csv

performance-consolidate:
needs:
- performance
runs-on: ubuntu-latest
if: startsWith(github.ref, 'refs/heads/master') || startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/feature') || startsWith(github.ref, 'refs/heads/next')
steps:
- name: Check out repository
uses: actions/checkout@v4
- name: Use Node.js
uses: actions/setup-node@v4
with:
node-version: 18.x
- uses: actions/download-artifact@v4
with:
name: performance-benchmark-solidbench
path: performance/benchmark-solidbench/combinations/combination_0/output/
- name: Process benchmark detailed results
run: npx -p @rubensworks/process-sparql-benchmark-results psbr csv ghbench performance/benchmark-solidbench/combinations/combination_0/output/ --overrideCombinationLabels SolidBench --total false --detailed true --name ghbench-detail.json
- name: Process benchmark total results
run: npx -p @rubensworks/process-sparql-benchmark-results psbr csv ghbench performance/benchmark-solidbench/combinations/combination_0/output/ --overrideCombinationLabels SolidBench --total true --detailed false --name ghbench-total.json
- name: Determine benchmark detailed results target directory name
run: echo "BENCHMARK_DATA_DIR_PATH_DETAIL=comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-detail" >> $GITHUB_ENV
- name: Determine benchmarks total results target directory name
run: echo "BENCHMARK_DATA_DIR_PATH_TOTAL=comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-total" >> $GITHUB_ENV
- name: Finalize benchmarks detailed results
uses: benchmark-action/github-action-benchmark@v1
with:
name: Benchmarks detailed results
tool: 'customSmallerIsBetter'
output-file-path: ghbench-detail.json
github-token: ${{ secrets.PAT }}
auto-push: true
alert-comment-cc-users: '@rubensworks'
gh-repository: 'github.com/comunica/comunica-performance-results'
gh-pages-branch: 'master'
benchmark-data-dir-path: ${{ env.BENCHMARK_DATA_DIR_PATH_DETAIL }}
- name: Cleanup benchmark-action data
run: rm -rf ./benchmark-data-repository
- name: Finalize benchmarks total results
uses: benchmark-action/github-action-benchmark@v1
with:
name: Benchmarks total results
tool: 'customSmallerIsBetter'
output-file-path: ghbench-total.json
github-token: ${{ secrets.PAT }}
auto-push: true
alert-threshold: '150%'
comment-always: true
comment-on-alert: true
alert-comment-cc-users: '@rubensworks'
summary-always: true
gh-repository: 'github.com/comunica/comunica-performance-results'
gh-pages-branch: 'master'
benchmark-data-dir-path: ${{ env.BENCHMARK_DATA_DIR_PATH_TOTAL }}
- name: Prepare comment on commit with link to performance results
run: echo -e "Performance benchmarks succeeded! 🚀\n\n\[[Summarized results](https://comunica.github.io/comunica-performance-results/comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-total/)\] \[[Detailed results](https://comunica.github.io/comunica-performance-results/comunica-feature-link-traversal/${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}/benchmarks-detail/)\]" > ./commit-comment-body.txt
- name: Comment on commit with link to performance results
uses: peter-evans/commit-comment@v3
with:
body-path: ./commit-comment-body.txt

docs:
runs-on: ubuntu-latest
steps:
Expand Down
1 change: 1 addition & 0 deletions eslint.config.js
Original file line number Diff line number Diff line change
Expand Up @@ -82,6 +82,7 @@ module.exports = config([
'**/web-clients/**',
'**/bintest/**',
'lerna.json',
'performance/**',
],
},
{
Expand Down
2 changes: 1 addition & 1 deletion lerna.json
Original file line number Diff line number Diff line change
Expand Up @@ -18,4 +18,4 @@
"loglevel": "success",
"registry": "https://registry.npmjs.org/",
"npmClient": "yarn"
}
}
1 change: 1 addition & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
"@pollyjs/core": "^6.0.6",
"@pollyjs/persister-fs": "^6.0.6",
"@rubensworks/eslint-config": "^3.0.0",
"@rubensworks/process-sparql-benchmark-results": "^1.3.3",
"@types/jest": "^29.5.5",
"@types/node": "^20.0.0",
"@types/setup-polly-jest": "^0.5.2",
Expand Down
11 changes: 11 additions & 0 deletions performance/benchmark-solidbench/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
/componentsjs-error-state.json
/node_modules
/generated/*
!/generated/.keep
/output/*
!/output/.keep
/combinations/*/generated/*
!/combinations/*/generated/.keep
/combinations/*/output/*
!/combinations/*/output/.keep
plot_queries_data.*
12 changes: 12 additions & 0 deletions performance/benchmark-solidbench/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# Benchmark SolidBench

This internal package benchmarks Comunica Link Traversal Solid using the [SolidBench](https://github.com/SolidBench/SolidBench.js) benchmark.

Compare your current version of Comunica locally with the latest published release by running `npm run performance` from within this package.
This will output a file called `plot_queries_data.svg` that visualizes the performance differences.

If you only want to check the performance of your current version of Comunica,
you can run `npm run performance:ci` instead,
which is what the CI will run as well for continuous performance measurements.

Continuous performance results are tracked on https://github.com/comunica/comunica-performance-results.
Empty file.
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,8 @@
{
"@context": [
"https://linkedsoftwaredependencies.org/bundles/npm/@comunica/config-query-sparql-link-traversal/^0.0.0/components/context.jsonld"
],
"import": [
"ccqslt:config/config-solid-default.json"
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/ldbc-snb-enhancer/^2.0.0/components/context.jsonld",
"@id": "urn:ldbc-snb-enhancer:default",
"@type": "Enhancer",
"personsPath": "out-snb/social_network/social_network_person_0_0.ttl",
"activitiesPath": "out-snb/social_network/social_network_activity_0_0.ttl",
"staticPath": "out-snb/social_network/social_network_static_0_0.ttl",
"destinationPathData": "out-enhanced/social_network_auxiliary.ttl",
"logger": {
"@type": "LoggerStdout"
},
"dataSelector": {
"@type": "DataSelectorRandom",
"seed": 12345
},
"parameterEmitterPosts": {
"@type": "ParameterEmitterCsv",
"destinationPath": "out-enhanced/parameters-posts.csv"
},
"handlers": [
{
"@type": "EnhancementHandlerPersonNoise",
"chance": 10
},
{
"@type": "EnhancementHandlerVocabulary"
}
]
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
{
"@context": "https://linkedsoftwaredependencies.org/bundles/npm/rdf-dataset-fragmenter/^2.0.0/components/context.jsonld",
"@id": "urn:rdf-dataset-fragmenter:default",
"@type": "Fragmenter",
"quadSource": {
"@id": "urn:rdf-dataset-fragmenter:source:default",
"@type": "QuadSourceComposite",
"sources": [
{
"@type": "QuadSourceFile",
"filePath": "out-enhanced/social_network_auxiliary.ttl"
}
]
},
"transformers": [
{
"@type": "QuadTransformerReplaceIri",
"searchRegex": "^http://www.ldbc.eu",
"replacementString": "http://solidbench-server:3000/www.ldbc.eu"
},
{
"@type": "QuadTransformerReplaceIri",
"searchRegex": "^http://dbpedia.org",
"replacementString": "http://solidbench-server:3000/dbpedia.org"
},
{
"@type": "QuadTransformerReplaceIri",
"searchRegex": "^http://www.w3.org/2002/07/owl",
"replacementString": "http://solidbench-server:3000/www.w3.org/2002/07/owl"
},
{
"@type": "QuadTransformerReplaceIri",
"searchRegex": "^http://solidbench-server:3000/www.ldbc.eu/ldbc_socialnet/1.0/data/pers([0-9]*)$",
"replacementString": "http://solidbench-server:3000/pods/$1/profile/card#me"
},
{
"@type": "QuadTransformerRemapResourceIdentifier",
"newIdentifierSeparator": "../noise/",
"typeRegex": "vocabulary/Noise$",
"identifierPredicateRegex": "vocabulary/noise$",
"targetPredicateRegex": "vocabulary/hasCreator$"
}
],
"fragmentationStrategy": {
"@type": "FragmentationStrategyComposite",
"strategies": [
{ "@type": "FragmentationStrategySubject" }
]
},
"quadSink": {
"@id": "urn:rdf-dataset-fragmenter:sink:default",
"@type": "QuadSinkFile",
"log": true,
"outputFormat": "application/n-quads",
"fileExtension": ".nq",
"iriToPath": {
"http://": "out-fragments/http/",
"https://": "out-fragments/https/"
}
}
}
Loading

3 comments on commit 6da1c4a

@rubensworks
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Benchmarks total results

Benchmark suite Current: 6da1c4a Previous: 2f6bcb6 Ratio
SolidBench 13844 ms 0 ms +∞

This comment was automatically generated by workflow using github-action-benchmark.

@rubensworks
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

⚠️ Performance Alert ⚠️

Possible performance regression was detected for benchmark 'Benchmarks total results'.
Benchmark result of this commit is worse than the previous benchmark result exceeding threshold 1.50.

Benchmark suite Current: 6da1c4a Previous: 2f6bcb6 Ratio
SolidBench 13844 ms 0 ms +∞

This comment was automatically generated by workflow using github-action-benchmark.

CC: @rubensworks

@github-actions
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Performance benchmarks succeeded! 🚀

[Summarized results] [Detailed results]

Please sign in to comment.