Skip to content

Commit ba8029e

Browse files
HyukjinKwonHyukjinKwon
committed
Migrate PR builders from Jenkins to Github Actions
1 parent 1cb5bfc commit ba8029e

File tree

20 files changed

+396
-160
lines changed

20 files changed

+396
-160
lines changed

‎.github/workflows/master.yml‎

Lines changed: 175 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -9,148 +9,231 @@ on:
99
- master
1010

1111
jobs:
12+
# Build: build Spark and run the tests for specified modules.
1213
build:
13-
14+
name: "Build modules: ${{ matrix.modules }} ${{ matrix.comment }} (JDK ${{ matrix.java }}, ${{ matrix.hadoop }}, ${{ matrix.hive }})"
1415
runs-on: ubuntu-latest
1516
strategy:
17+
fail-fast: false
1618
matrix:
17-
java: [ '1.8', '11' ]
18-
hadoop: [ 'hadoop-2.7', 'hadoop-3.2' ]
19-
hive: [ 'hive-1.2', 'hive-2.3' ]
20-
exclude:
21-
- java: '11'
22-
hive: 'hive-1.2'
23-
- hadoop: 'hadoop-3.2'
24-
hive: 'hive-1.2'
25-
name: Build Spark - JDK${{ matrix.java }}/${{ matrix.hadoop }}/${{ matrix.hive }}
26-
19+
java:
20+
- 1.8
21+
hadoop:
22+
- hadoop3.2
23+
hive:
24+
- hive2.3
25+
# TODO(SPARK-XXXXX): We don't test 'streaming-kinesis-asl' for now.
26+
# Kinesis tests depends on external Amazon kinesis service.
27+
# Note that the modules below are from sparktestsupport/modules.py.
28+
modules:
29+
- |-
30+
core, unsafe, kvstore, avro,
31+
network_common, network_shuffle, repl, launcher
32+
examples, sketch, graphx
33+
- |-
34+
catalyst, sql
35+
- |-
36+
hive-thriftserver
37+
- |-
38+
streaming, sql-kafka-0-10, streaming-kafka-0-10
39+
- |-
40+
mllib-local, mllib
41+
- |-
42+
pyspark-sql, pyspark-mllib, pyspark-resource
43+
- |-
44+
pyspark-core, pyspark-streaming, pyspark-ml
45+
- |-
46+
sparkr
47+
- |-
48+
yarn, mesos, kubernetes, hadoop-cloud,
49+
spark-ganglia-lgpl
50+
# Here, we split Hive tests into some of heavy ones and the rest of them.
51+
included-tags: [""]
52+
excluded-tags: [""]
53+
comment: ["- running all tests"]
54+
include:
55+
- modules: hive
56+
java: 1.8
57+
hadoop: hadoop3.2
58+
hive: hive2.3
59+
included-tags: org.apache.spark.tags.HeavyHiveTest
60+
comment: "- running heavy tests"
61+
- modules: hive
62+
java: 1.8
63+
hadoop: hadoop3.2
64+
hive: hive2.3
65+
excluded-tags: org.apache.spark.tags.HeavyHiveTest
66+
comment: "- running non-heavy tests"
67+
env:
68+
TEST_ONLY_MODULES: ${{ matrix.modules }}
69+
HADOOP_PROFILE: ${{ matrix.hadoop }}
70+
HIVE_PROFILE: ${{ matrix.hive }}
71+
# Github Actions' default miniconda
72+
CONDA_PREFIX: /usr/share/miniconda
73+
# Don't run the tests in parallel due to flakiness. See SparkParallelTestGrouping.
74+
SERIAL_SBT_TESTS: 1
75+
TEST_ONLY_EXCLUDED_TAGS: ${{ matrix.excluded-tags }}
76+
TEST_ONLY_INCLUDED_TAGS: ${{ matrix.included-tags }}
2777
steps:
28-
- uses: actions/checkout@master
29-
# We split caches because GitHub Action Cache has a 400MB-size limit.
30-
- uses: actions/cache@v1
78+
- name: Checkout Spark repository
79+
uses: actions/checkout@v2
80+
# Cache local repositories. Note that Github Actions cache has a 2G limit.
81+
- name: Cache Scala, SBT, Maven and Zinc
82+
uses: actions/cache@v1
3183
with:
3284
path: build
3385
key: build-${{ hashFiles('**/pom.xml') }}
3486
restore-keys: |
3587
build-
36-
- uses: actions/cache@v1
37-
with:
38-
path: ~/.m2/repository/com
39-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-${{ hashFiles('**/pom.xml') }}
40-
restore-keys: |
41-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-com-
42-
- uses: actions/cache@v1
88+
- name: Cache Maven local repository
89+
uses: actions/cache@v2
4390
with:
44-
path: ~/.m2/repository/org
45-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-${{ hashFiles('**/pom.xml') }}
46-
restore-keys: |
47-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-org-
48-
- uses: actions/cache@v1
49-
with:
50-
path: ~/.m2/repository/net
51-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-${{ hashFiles('**/pom.xml') }}
91+
path: ~/.m2/repository
92+
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-${{ hashFiles('**/pom.xml') }}
5293
restore-keys: |
53-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-net-
54-
- uses: actions/cache@v1
94+
${{ matrix.java }}-${{ matrix.hadoop }}-maven-
95+
- name: Cache Ivy local repository
96+
uses: actions/cache@v2
5597
with:
56-
path: ~/.m2/repository/io
57-
key: ${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-${{ hashFiles('**/pom.xml') }}
98+
path: ~/.ivy2/cache
99+
key: ${{ matrix.java }}-${{ matrix.hadoop }}-ivy-${{ hashFiles('**/pom.xml') }}-${{ hashFiles('**/plugins.sbt') }}
58100
restore-keys: |
59-
${{ matrix.java }}-${{ matrix.hadoop }}-maven-io-
60-
- name: Set up JDK ${{ matrix.java }}
101+
${{ matrix.java }}-${{ matrix.hadoop }}-ivy-
102+
- name: Install JDK ${{ matrix.java }}
61103
uses: actions/setup-java@v1
62104
with:
63105
java-version: ${{ matrix.java }}
64-
- name: Build with Maven
106+
# PySpark
107+
- name: Install PyPy3
108+
# SQL component also has Python related tests, for example, IntegratedUDFTestUtils.
109+
# Note that order here matters because default python3 is overridden by pypy3.
110+
uses: actions/setup-python@v2
111+
if: contains(matrix.modules, 'pyspark') || matrix.modules == 'sql'
112+
with:
113+
python-version: pypy3
114+
architecture: x64
115+
- name: Install Python 2.7
116+
uses: actions/setup-python@v2
117+
if: contains(matrix.modules, 'pyspark') || matrix.modules == 'sql'
118+
with:
119+
python-version: 2.7
120+
architecture: x64
121+
- name: Install Python 3.6
122+
uses: actions/setup-python@v2
123+
if: contains(matrix.modules, 'pyspark') || matrix.modules == 'sql'
124+
with:
125+
python-version: 3.6
126+
architecture: x64
127+
- name: Install Python packages
128+
if: contains(matrix.modules, 'pyspark') || matrix.modules == 'sql'
129+
# PyArrow is not supported in PyPy yet, see ARROW-2651.
130+
# scipy installation with PyPy fails for an unknown reason.
131+
run: |
132+
python3 -m pip install numpy pyarrow pandas scipy
133+
python3 -m pip list
134+
python2 -m pip install numpy pyarrow pandas scipy
135+
python2 -m pip list
136+
# Installing NumPy is flaky in PyPy.
137+
pypy3 -m pip install numpy pandas
138+
pypy3 -m pip list
139+
# SparkR
140+
- name: Install R 3.6
141+
uses: r-lib/actions/setup-r@v1
142+
if: contains(matrix.modules, 'sparkr')
143+
with:
144+
r-version: 3.6
145+
- name: Install R packages
146+
if: contains(matrix.modules, 'sparkr')
147+
run: |
148+
sudo apt-get install -y libcurl4-openssl-dev
149+
sudo Rscript -e "install.packages(c('knitr', 'rmarkdown', 'testthat', 'devtools', 'e1071', 'survival', 'arrow', 'roxygen2'), repos='https://cloud.r-project.org/')"
150+
# Show installed packages in R.
151+
sudo Rscript -e 'pkg_list <- as.data.frame(installed.packages()[, c(1,3:4)]); pkg_list[is.na(pkg_list$Priority), 1:2, drop = FALSE]'
152+
# Run the tests.
153+
- name: "Run tests: ${{ matrix.modules }}"
65154
run: |
66-
export MAVEN_OPTS="-Xmx2g -XX:ReservedCodeCacheSize=1g -Dorg.slf4j.simpleLogger.defaultLogLevel=WARN"
67-
export MAVEN_CLI_OPTS="--no-transfer-progress"
68155
mkdir -p ~/.m2
69-
./build/mvn $MAVEN_CLI_OPTS -DskipTests -Pyarn -Pmesos -Pkubernetes -Phive -P${{ matrix.hive }} -Phive-thriftserver -P${{ matrix.hadoop }} -Phadoop-cloud -Djava.version=${{ matrix.java }} install
156+
./dev/run-tests --parallelism 2
70157
rm -rf ~/.m2/repository/org/apache/spark
71158
72-
159+
# Linters: run the linters and other static analysis.
73160
lint:
161+
name: Linters, licenses, dependencies
74162
runs-on: ubuntu-latest
75-
name: Linters (Java/Scala/Python), licenses, dependencies
76163
steps:
77164
- uses: actions/checkout@master
78-
- uses: actions/setup-java@v1
165+
- name: Install JDK 11
166+
uses: actions/setup-java@v1
79167
with:
80-
java-version: '11'
81-
- uses: actions/setup-python@v1
168+
java-version: 11
169+
- name: Install Python 3.6
170+
uses: actions/setup-python@v2
82171
with:
83-
python-version: '3.x'
84-
architecture: 'x64'
85-
- name: Scala
86-
run: ./dev/lint-scala
87-
- name: Java
88-
run: ./dev/lint-java
89-
- name: Python
172+
python-version: 3.6
173+
architecture: x64
174+
- name: Install Python linter dependencies
90175
run: |
91-
pip install flake8 sphinx numpy
92-
./dev/lint-python
93-
- name: License
94-
run: ./dev/check-license
95-
- name: Dependencies
96-
run: ./dev/test-dependencies.sh
97-
98-
lintr:
99-
runs-on: ubuntu-latest
100-
name: Linter (R)
101-
steps:
102-
- uses: actions/checkout@master
103-
- uses: actions/setup-java@v1
176+
pip3 install flake8 sphinx numpy
177+
- name: Install R 3.6
178+
uses: r-lib/actions/setup-r@v1
104179
with:
105-
java-version: '11'
106-
- uses: r-lib/actions/setup-r@v1
107-
with:
108-
r-version: '3.6.2'
109-
- name: Install lib
180+
r-version: 3.6
181+
- name: Install R linter dependencies and SparkR
110182
run: |
111183
sudo apt-get install -y libcurl4-openssl-dev
112-
- name: install R packages
113-
run: |
114-
sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
184+
sudo Rscript -e "install.packages(c('devtools'), repos='https://cloud.r-project.org/')"
115185
sudo Rscript -e "devtools::install_github('jimhester/lintr@v2.0.0')"
116-
- name: package and install SparkR
117-
run: ./R/install-dev.sh
118-
- name: lint-r
186+
./R/install-dev.sh
187+
- name: Scala linter
188+
run: ./dev/lint-scala
189+
- name: Java linter
190+
run: ./dev/lint-java
191+
- name: Python linter
192+
run: ./dev/lint-python
193+
- name: R linter
119194
run: ./dev/lint-r
195+
- name: License test
196+
run: ./dev/check-license
197+
- name: Dependencies test
198+
run: ./dev/test-dependencies.sh
120199

200+
# Documentation build.
121201
docs:
202+
name: Build documentation
122203
runs-on: ubuntu-latest
123-
name: Generate documents
124204
steps:
125205
- uses: actions/checkout@master
126-
- uses: actions/cache@v1
206+
- name: Cache Maven local repository
207+
uses: actions/cache@v2
127208
with:
128209
path: ~/.m2/repository
129210
key: docs-maven-repo-${{ hashFiles('**/pom.xml') }}
130211
restore-keys: |
131-
docs-maven-repo-
132-
- uses: actions/setup-java@v1
212+
docs-maven-
213+
- name: Install JDK 1.8
214+
uses: actions/setup-java@v1
133215
with:
134-
java-version: '1.8'
135-
- uses: actions/setup-python@v1
216+
java-version: 1.8
217+
- name: Install Python 3.6
218+
uses: actions/setup-python@v2
136219
with:
137-
python-version: '3.x'
138-
architecture: 'x64'
139-
- uses: actions/setup-ruby@v1
220+
python-version: 3.6
221+
architecture: x64
222+
- name: Install Ruby 2.7
223+
uses: actions/setup-ruby@v1
140224
with:
141-
ruby-version: '2.7'
142-
- uses: r-lib/actions/setup-r@v1
225+
ruby-version: 2.7
226+
- name: Install R 3.6
227+
uses: r-lib/actions/setup-r@v1
143228
with:
144-
r-version: '3.6.2'
145-
- name: Install lib and pandoc
229+
r-version: 3.6
230+
- name: Install dependencies
146231
run: |
147232
sudo apt-get install -y libcurl4-openssl-dev pandoc
148-
- name: Install packages
149-
run: |
150233
pip install sphinx mkdocs numpy
151234
gem install jekyll jekyll-redirect-from rouge
152-
sudo Rscript -e "install.packages(c('curl', 'xml2', 'httr', 'devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2', 'e1071', 'survival'), repos='https://cloud.r-project.org/')"
153-
- name: Run jekyll build
235+
sudo Rscript -e "install.packages(c('devtools', 'testthat', 'knitr', 'rmarkdown', 'roxygen2'), repos='https://cloud.r-project.org/')"
236+
- name: Run documentation build
154237
run: |
155238
cd docs
156239
jekyll build

‎common/tags/src/test/java/org/apache/spark/tags/HeavyHiveTest.java‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.spark.tags;
19+
20+
import org.scalatest.TagAnnotation;
21+
22+
import java.lang.annotation.ElementType;
23+
import java.lang.annotation.Retention;
24+
import java.lang.annotation.RetentionPolicy;
25+
import java.lang.annotation.Target;
26+
27+
@TagAnnotation
28+
@Retention(RetentionPolicy.RUNTIME)
29+
@Target({ElementType.METHOD, ElementType.TYPE})
30+
public @interface HeavyHiveTest { }

‎core/src/test/scala/org/apache/spark/deploy/master/MasterSuite.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -685,7 +685,7 @@ class MasterSuite extends SparkFunSuite
685685
}
686686
}
687687

688-
test("SPARK-27510: Master should avoid dead loop while launching executor failed in Worker") {
688+
ignore("SPARK-27510: Master should avoid dead loop while launching executor failed in Worker") {
689689
val master = makeAliveMaster()
690690
var worker: MockExecutorLaunchFailWorker = null
691691
try {

‎dev/run-pip-tests‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,8 +76,12 @@ for python in "${PYTHON_EXECS[@]}"; do
7676
VIRTUALENV_PATH="$VIRTUALENV_BASE"/$python
7777
rm -rf "$VIRTUALENV_PATH"
7878
if [ -n "$USE_CONDA" ]; then
79+
if [ -f "$CONDA_PREFIX/etc/profile.d/conda.sh" ]; then
80+
# See also https://github.com/conda/conda/issues/7980
81+
source $CONDA_PREFIX/etc/profile.d/conda.sh
82+
fi
7983
conda create -y -p "$VIRTUALENV_PATH" python=$python numpy pandas pip setuptools
80-
source activate "$VIRTUALENV_PATH"
84+
conda activate "$VIRTUALENV_PATH"
8185
else
8286
mkdir -p "$VIRTUALENV_PATH"
8387
virtualenv --python=$python "$VIRTUALENV_PATH"
@@ -120,7 +124,7 @@ for python in "${PYTHON_EXECS[@]}"; do
120124

121125
# conda / virtualenv environments need to be deactivated differently
122126
if [ -n "$USE_CONDA" ]; then
123-
source deactivate
127+
conda deactivate
124128
else
125129
deactivate
126130
fi

0 commit comments

Comments
 (0)