Skip to content

Commit c1792af

Browse files
alhuelamoalhuelamo
authored
Merge pull request #1 from alhuelamo/dev/0.1.0
Dev/0.1.0
2 parents d51eb28 + 3994734 commit c1792af

File tree

12 files changed

+368
-62
lines changed

12 files changed

+368
-62
lines changed

‎.github/workflows/ci.yml

+119
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,119 @@
1+
# This file was automatically generated by sbt-github-actions using the
2+
# githubWorkflowGenerate task. You should add and commit this file to
3+
# your git repository. It goes without saying that you shouldn't edit
4+
# this file by hand! Instead, if you wish to make changes, you should
5+
# change your sbt build configuration to revise the workflow description
6+
# to meet your needs, then regenerate this file.
7+
8+
name: Continuous Integration
9+
10+
on:
11+
pull_request:
12+
branches: ['**']
13+
push:
14+
branches: ['**']
15+
tags: [v*]
16+
17+
env:
18+
PGP_PASSPHRASE: ${{ secrets.PGP_PASSPHRASE }}
19+
SONATYPE_PASSWORD: ${{ secrets.SONATYPE_PASSWORD }}
20+
SONATYPE_USERNAME: ${{ secrets.SONATYPE_USERNAME }}
21+
PGP_SECRET: ${{ secrets.PGP_SECRET }}
22+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
23+
24+
jobs:
25+
build:
26+
name: Build and Test
27+
strategy:
28+
matrix:
29+
os: [ubuntu-latest]
30+
scala: [3.1.1]
31+
java: [graal_21.3.0@11]
32+
runs-on: ${{ matrix.os }}
33+
steps:
34+
- name: Checkout current branch (full)
35+
uses: actions/checkout@v2
36+
with:
37+
fetch-depth: 0
38+
39+
- name: Setup GraalVM (graal_21.3.0@11)
40+
if: matrix.java == 'graal_21.3.0@11'
41+
uses: DeLaGuardo/setup-graalvm@5.0
42+
with:
43+
graalvm: 21.3.0
44+
java: java11
45+
46+
- name: Cache sbt
47+
uses: actions/cache@v2
48+
with:
49+
path: |
50+
~/.sbt
51+
~/.ivy2/cache
52+
~/.coursier/cache/v1
53+
~/.cache/coursier/v1
54+
~/AppData/Local/Coursier/Cache/v1
55+
~/Library/Caches/Coursier/v1
56+
key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
57+
58+
- name: Check that workflows are up to date
59+
run: sbt ++${{ matrix.scala }} githubWorkflowCheck
60+
61+
- name: Build project
62+
run: sbt ++${{ matrix.scala }} test
63+
64+
- name: Compress target directories
65+
run: tar cf targets.tar target project/target
66+
67+
- name: Upload target directories
68+
uses: actions/upload-artifact@v2
69+
with:
70+
name: target-${{ matrix.os }}-${{ matrix.scala }}-${{ matrix.java }}
71+
path: targets.tar
72+
73+
publish:
74+
name: Publish Artifacts
75+
needs: [build]
76+
if: github.event_name != 'pull_request' && (startsWith(github.ref, 'refs/tags/v') || github.ref == 'refs/heads/main')
77+
strategy:
78+
matrix:
79+
os: [ubuntu-latest]
80+
scala: [3.1.1]
81+
java: [graal_21.3.0@11]
82+
runs-on: ${{ matrix.os }}
83+
steps:
84+
- name: Checkout current branch (full)
85+
uses: actions/checkout@v2
86+
with:
87+
fetch-depth: 0
88+
89+
- name: Setup GraalVM (graal_21.3.0@11)
90+
if: matrix.java == 'graal_21.3.0@11'
91+
uses: DeLaGuardo/setup-graalvm@5.0
92+
with:
93+
graalvm: 21.3.0
94+
java: java11
95+
96+
- name: Cache sbt
97+
uses: actions/cache@v2
98+
with:
99+
path: |
100+
~/.sbt
101+
~/.ivy2/cache
102+
~/.coursier/cache/v1
103+
~/.cache/coursier/v1
104+
~/AppData/Local/Coursier/Cache/v1
105+
~/Library/Caches/Coursier/v1
106+
key: ${{ runner.os }}-sbt-cache-v2-${{ hashFiles('**/*.sbt') }}-${{ hashFiles('project/build.properties') }}
107+
108+
- name: Download target directories (3.1.1)
109+
uses: actions/download-artifact@v2
110+
with:
111+
name: target-${{ matrix.os }}-3.1.1-${{ matrix.java }}
112+
113+
- name: Inflate target directories (3.1.1)
114+
run: |
115+
tar xf targets.tar
116+
rm targets.tar
117+
118+
- id: release
119+
run: sbt ++${{ matrix.scala }} ci-release

‎.github/workflows/clean.yml

+59
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
# This file was automatically generated by sbt-github-actions using the
2+
# githubWorkflowGenerate task. You should add and commit this file to
3+
# your git repository. It goes without saying that you shouldn't edit
4+
# this file by hand! Instead, if you wish to make changes, you should
5+
# change your sbt build configuration to revise the workflow description
6+
# to meet your needs, then regenerate this file.
7+
8+
name: Clean
9+
10+
on: push
11+
12+
jobs:
13+
delete-artifacts:
14+
name: Delete Artifacts
15+
runs-on: ubuntu-latest
16+
env:
17+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
18+
steps:
19+
- name: Delete artifacts
20+
run: |
21+
# Customize those three lines with your repository and credentials:
22+
REPO=${GITHUB_API_URL}/repos/${{ github.repository }}
23+
24+
# A shortcut to call GitHub API.
25+
ghapi() { curl --silent --location --user _:$GITHUB_TOKEN "$@"; }
26+
27+
# A temporary file which receives HTTP response headers.
28+
TMPFILE=/tmp/tmp.$$
29+
30+
# An associative array, key: artifact name, value: number of artifacts of that name.
31+
declare -A ARTCOUNT
32+
33+
# Process all artifacts on this repository, loop on returned "pages".
34+
URL=$REPO/actions/artifacts
35+
while [[ -n "$URL" ]]; do
36+
37+
# Get current page, get response headers in a temporary file.
38+
JSON=$(ghapi --dump-header $TMPFILE "$URL")
39+
40+
# Get URL of next page. Will be empty if we are at the last page.
41+
URL=$(grep '^Link:' "$TMPFILE" | tr ',' '\n' | grep 'rel="next"' | head -1 | sed -e 's/.*<//' -e 's/>.*//')
42+
rm -f $TMPFILE
43+
44+
# Number of artifacts on this page:
45+
COUNT=$(( $(jq <<<$JSON -r '.artifacts | length') ))
46+
47+
# Loop on all artifacts on this page.
48+
for ((i=0; $i < $COUNT; i++)); do
49+
50+
# Get name of artifact and count instances of this name.
51+
name=$(jq <<<$JSON -r ".artifacts[$i].name?")
52+
ARTCOUNT[$name]=$(( $(( ${ARTCOUNT[$name]} )) + 1))
53+
54+
id=$(jq <<<$JSON -r ".artifacts[$i].id?")
55+
size=$(( $(jq <<<$JSON -r ".artifacts[$i].size_in_bytes?") ))
56+
printf "Deleting '%s' #%d, %'d bytes\n" $name ${ARTCOUNT[$name]} $size
57+
ghapi -X DELETE $REPO/actions/artifacts/$id
58+
done
59+
done

‎LICENSE

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
2+
3+
4+
Copyright 2021 Alberto Huélamo
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.

‎README.md

+56
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,68 @@ It has basically two commands:
1010

1111
## Installation
1212

13+
If you use [Coursier](https://get-coursier.io) you can install the application in your current working directory by running
1314

15+
```bash
16+
coursier bootstrap com.alhuelamo:dbjobs_3:0.1.0 -o dbjobs
17+
```
1418

1519
## Building from source
1620

21+
You will need [sbt](https://www.scala-sbt.org) as the main pre-requisite.
1722

23+
Go to the repository folder and run
24+
25+
```bash
26+
sbt stage
27+
```
28+
29+
to generate a launcher file for the app. Then you can go to the folder
30+
31+
```bash
32+
cd ./target/universal/stage
33+
```
34+
35+
and run
36+
37+
```bash
38+
./bin/dbjobs --help
39+
```
1840

1941
## Usage
2042

43+
The application requires the presence of a [Databricks CLI config file](https://docs.databricks.com/dev-tools/cli/index.html) in your home folder (`~/.databrickscfg`).
44+
45+
You need to define a profile for the Databricks workspace you want to point at.
46+
47+
```bash
48+
vim ~/.databrickscfg
49+
```
50+
51+
```ini
52+
[myprofile]
53+
host=https://mydatabricks.workspace.url.com
54+
token=myapiaccesstoken
55+
```
56+
57+
Once configured, you can use the app to start and stop existing jobs in that workspace.
58+
59+
Start jobs
60+
61+
```bash
62+
dbjobs start \
63+
--profile myprofile \
64+
--job-ids 42,314,9
65+
```
66+
67+
Stop active jobs:
68+
69+
```bash
70+
dbjobs stop \
71+
--profile myprofile \
72+
--job-ids 42,314,9
73+
```
74+
75+
### Planning
2176

77+
You can use the flag `--plan` to just show which are going to be the affected jobs and [runs](https://docs.databricks.com/dev-tools/api/latest/jobs.html#operation/JobsRunsList) without actually starting or stopping them.

‎build.sbt

+27
Original file line numberDiff line numberDiff line change
@@ -12,8 +12,35 @@ inThisBuild(List(
1212
),
1313
scalaVersion := "3.1.1",
1414
versionScheme := Some("early-semver"),
15+
// Sonatype location
16+
sonatypeCredentialHost := "s01.oss.sonatype.org",
17+
sonatypeRepository := "https://s01.oss.sonatype.org/service/local"
1518
))
1619

20+
// CI/CD generation
21+
ThisBuild / githubWorkflowJavaVersions := Seq(JavaSpec.graalvm("21.3.0", "11"))
22+
ThisBuild / githubWorkflowTargetTags := Seq("v*")
23+
ThisBuild / githubWorkflowPublishTargetBranches := List(
24+
RefPredicate.StartsWith(Ref.Tag("v")),
25+
RefPredicate.Equals(Ref.Branch("main"))
26+
)
27+
28+
ThisBuild / githubWorkflowPublish := Seq(
29+
WorkflowStep.Sbt(
30+
List("ci-release"),
31+
id = Some("release")
32+
)
33+
)
34+
35+
ThisBuild / githubWorkflowEnv ++= List(
36+
"PGP_PASSPHRASE",
37+
"PGP_SECRET",
38+
"SONATYPE_PASSWORD",
39+
"SONATYPE_USERNAME"
40+
).map { envKey =>
41+
envKey -> s"$${{ secrets.$envKey }}"
42+
}.toMap
43+
1744
lazy val root = project
1845
.in(file("."))
1946
.enablePlugins(JavaAppPackaging, NativeImagePlugin)

‎src/main/scala/com/alhuelamo/databricks/jobmanager/Actions.scala

+15-19
Original file line numberDiff line numberDiff line change
@@ -11,51 +11,47 @@ object Actions {
1111
val ws = conf.databricksWs
1212
println(s"Stopping job $jobId")
1313

14-
Try {
14+
try {
1515
val activeRuns = DatabricksApi.getActiveJobRuns(jobId, ws)
1616
if (activeRuns.isEmpty)
1717
println(" no active runs found for this job.")
1818
else
1919
activeRuns.foreach(runId => cancelRun(runId, ws))
20-
} recover {
21-
jobNotFound
20+
} catch {
21+
handleApiErrors("job")
2222
}
2323
}
2424

2525
private def cancelRun(runId: Long, ws: DatabricksWorkspace)(using conf: AppConf): Unit = {
2626
println(s" on run $runId")
2727

2828
if (!conf.plan) {
29-
Try {
29+
try {
3030
DatabricksApi.cancelJobRun(runId, ws)
31-
} recover {
32-
runNotFound
31+
} catch {
32+
handleApiErrors("run")
3333
}
3434
}
3535
}
3636

3737
def startRuns(jobId: Long)(using conf: AppConf): Unit = {
3838
println(s"Starting job $jobId")
3939
if (!conf.plan) {
40-
Try {
40+
try {
4141
DatabricksApi.triggerJobRun(jobId, conf.databricksWs)
42-
} recover {
43-
jobNotFound
42+
} catch {
43+
handleApiErrors("job")
4444
}
4545
}
4646
}
4747

48-
private val jobNotFound: PartialFunction[Throwable, Unit] = {
49-
case ApiException(_, 400) => println(" job not found!")
48+
private def handleApiErrors(resource: String): PartialFunction[Throwable, Unit] = {
49+
case ApiException(_, 400) => println(s" $resource not found!")
50+
case ex @ ApiException(_, 403) =>
51+
println(" authentication error!")
52+
throw ex
5053
case NonFatal(error) =>
51-
println(" problem querying the job!")
52-
throw error
53-
}
54-
55-
private val runNotFound: PartialFunction[Throwable, Unit] = {
56-
case ApiException(_, 400) => println(s" run id not found!")
57-
case NonFatal(error) =>
58-
println(" problem cancelling the run!")
54+
println(s" problem querying the $resource!")
5955
throw error
6056
}
6157

0 commit comments

Comments
 (0)