Skip to content
Closed
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 26 additions & 28 deletions js/downloads.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@

releases = {};

function addRelease(version, releaseDate, packages, stable) {
releases[version] = {released: releaseDate, packages: packages, stable: stable};
function addRelease(version, releaseDate, packages, stable, mirrored) {
releases[version] = {
released: releaseDate,
packages: packages,
stable: stable,
mirrored: mirrored
};
}

var sources = {pretty: "Source Code", tag: "sources"};
Expand All @@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source
// 2.2.0+
var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources];

addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true);
addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false);
//addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true);
//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true);
//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true);
//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true);
//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true);
//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false);

function append(el, contents) {
el.innerHTML += contents;
Expand Down Expand Up @@ -128,10 +128,10 @@ function onVersionSelect() {
append(verifyLink, link);

// Populate releases
updateDownloadLink();
updateDownloadLink(releases[version].mirrored);
}

function updateDownloadLink() {
function updateDownloadLink(isMirrored) {
var versionSelect = document.getElementById("sparkVersionSelect");
var packageSelect = document.getElementById("sparkPackageSelect");
var downloadLink = document.getElementById("spanDownloadLink");
Expand All @@ -147,12 +147,10 @@ function updateDownloadLink() {
.replace(/-bin-sources/, ""); // special case for source packages

var link = "";
if (version < "1.6.3" ||
(version >= "2.0.0" && version <= "2.0.1") ||
(version >= "2.1.0" && version <= "2.1.1")) {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
} else {
if (isMirrored) {
link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact";
} else {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
}
link = link
.replace(/\$ver/, version)
Expand Down
11 changes: 10 additions & 1 deletion release-process.md
Original file line number Diff line number Diff line change
Expand Up @@ -194,9 +194,18 @@ You should also delete the RC directories from the staging repository. For examp
svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
-m"Removing RC artifacts."

```

Make sure to also remove the unpublished staging repositories from the <a href="https://repository.apache.org/">Apache Nexus Repository Manager</a>.

When removing older releases, also update `js/download.js` to indicate the release is not mirrored
anymore, so that the correct links are generated on the site.

Also take a moment to check `HiveExternalCatalogVersionsSuite.scala` starting with branch-2.2
and see if it needs to be adjusted, since that test relies on mirrored downloads of previous
releases.


<h4>Update the Spark Apache Repository</h4>

Check out the tagged commit for the release candidate that passed and apply the correct version tag.
Expand Down
5 changes: 4 additions & 1 deletion site/doap.rdf
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@
<homepage rdf:resource="http://spark.apache.org" />
<asfext:pmc rdf:resource="http://spark.apache.org" />
<shortdesc>Apache Spark is a fast and general engine for large-scale data processing.</shortdesc>
<description>Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala and Python as well as a rich set of libraries including stream processing, machine learning, and graph analytics.</description>
<description>Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala, Python and R, as well as a rich set of libraries including stream processing, machine learning, and graph analytics.</description>
<bug-database rdf:resource="http://issues.apache.org/jira/browse/SPARK" />
<mailing-list rdf:resource="http://spark.apache.org/community.html" />
<download-page rdf:resource="http://spark.apache.org/downloads.html" />
<programming-language>Java</programming-language>
<programming-language>Scala</programming-language>
<programming-language>Python</programming-language>
<programming-language>R</programming-language>
<programming-language>SQL</programming-language>
<programming-language>JDBC</programming-language>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

<category rdf:resource="http://projects.apache.org/category/big-data" />
</Project>
</rdf:RDF>
54 changes: 26 additions & 28 deletions site/js/downloads.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@

releases = {};

function addRelease(version, releaseDate, packages, stable) {
releases[version] = {released: releaseDate, packages: packages, stable: stable};
function addRelease(version, releaseDate, packages, stable, mirrored) {
releases[version] = {
released: releaseDate,
packages: packages,
stable: stable,
mirrored: mirrored
};
}

var sources = {pretty: "Source Code", tag: "sources"};
Expand All @@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source
// 2.2.0+
var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources];

addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true);
addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false);
//addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true);
//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true);
//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true);
//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true);
//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true);
//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false);

function append(el, contents) {
el.innerHTML += contents;
Expand Down Expand Up @@ -128,10 +128,10 @@ function onVersionSelect() {
append(verifyLink, link);

// Populate releases
updateDownloadLink();
updateDownloadLink(releases[version].mirrored);
}

function updateDownloadLink() {
function updateDownloadLink(isMirrored) {
var versionSelect = document.getElementById("sparkVersionSelect");
var packageSelect = document.getElementById("sparkPackageSelect");
var downloadLink = document.getElementById("spanDownloadLink");
Expand All @@ -147,12 +147,10 @@ function updateDownloadLink() {
.replace(/-bin-sources/, ""); // special case for source packages

var link = "";
if (version < "1.6.3" ||
(version >= "2.0.0" && version <= "2.0.1") ||
(version >= "2.1.0" && version <= "2.1.1")) {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
} else {
if (isMirrored) {
link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact";
} else {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
}
link = link
.replace(/\$ver/, version)
Expand Down
10 changes: 9 additions & 1 deletion site/release-process.html
Original file line number Diff line number Diff line change
Expand Up @@ -391,9 +391,17 @@ <h4>Remove Old Releases from Development Repository and Mirror Network</h4>
<pre><code>svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
-m"Removing RC artifacts."

</code></pre>

<p>Make sure to also remove the unpublished staging repositories from the <a href="https://repository.apache.org/">Apache Nexus Repository Manager</a>.</p>

<p>When removing older releases, also update <code>js/download.js</code> to indicate the release is not mirrored
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could we expand on When removing older releases happens?
I think given major.minor.patch, the previous patch release is archived when a newer one in the same major.minor is released?
ie, 2.1.3 released, 2.1.2 archived

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is already sort of covered by the existing text:

Spark always keeps two releases in the mirror network: the most recent release on the current and
previous branches.

That doesn't seem to really correspond to the current mirrored releases (and with the expectation of the Hive tests that were added in 2.2), though, so maybe we should update that policy.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

right, that doesn't seem to match the ASF mirror behavior now

anymore, so that the correct links are generated on the site.</p>

<p>Also take a moment to check <code>HiveExternalCatalogVersionsSuite.scala</code> starting with branch-2.2
and see if it needs to be adjusted, since that test relies on mirrored downloads of previous
releases.</p>

<h4>Update the Spark Apache Repository</h4>

<p>Check out the tagged commit for the release candidate that passed and apply the correct version tag.</p>
Expand Down