Skip to content

Update list of mirrored releases, add more instructions to release guide. #117

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 3 commits into from
Closed

Update list of mirrored releases, add more instructions to release guide. #117

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
ca0128e
Update list of mirrored releases, add more instructions to release gu…
Jun 12, 2018
e963c61
More stuff.
Jun 12, 2018
58e9237
Better guidance for what to delete.
Jun 13, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
54 changes: 26 additions & 28 deletions js/downloads.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@

releases = {};

function addRelease(version, releaseDate, packages, stable) {
releases[version] = {released: releaseDate, packages: packages, stable: stable};
function addRelease(version, releaseDate, packages, stable, mirrored) {
releases[version] = {
released: releaseDate,
packages: packages,
stable: stable,
mirrored: mirrored
};
}

var sources = {pretty: "Source Code", tag: "sources"};
Expand All @@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source
// 2.2.0+
var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources];

addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true);
addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false);
//addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true);
//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true);
//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true);
//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true);
//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true);
//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false);

function append(el, contents) {
el.innerHTML += contents;
Expand Down Expand Up @@ -128,10 +128,10 @@ function onVersionSelect() {
append(verifyLink, link);

// Populate releases
updateDownloadLink();
updateDownloadLink(releases[version].mirrored);
}

function updateDownloadLink() {
function updateDownloadLink(isMirrored) {
var versionSelect = document.getElementById("sparkVersionSelect");
var packageSelect = document.getElementById("sparkPackageSelect");
var downloadLink = document.getElementById("spanDownloadLink");
Expand All @@ -147,12 +147,10 @@ function updateDownloadLink() {
.replace(/-bin-sources/, ""); // special case for source packages

var link = "";
if (version < "1.6.3" ||
(version >= "2.0.0" && version <= "2.0.1") ||
(version >= "2.1.0" && version <= "2.1.1")) {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
} else {
if (isMirrored) {
link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact";
} else {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
}
link = link
.replace(/\$ver/, version)
Expand Down
34 changes: 23 additions & 11 deletions release-process.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,26 +177,38 @@ Publishing to CRAN is done using <a href="https://cran.r-project.org/submit.html
Since it requires further manual steps, please also contact the <a href="mailto:private@spark.apache.org">PMC</a>.


<h4>Remove Old Releases from Development Repository and Mirror Network</h4>
<h4> Remove RC artifacts from repositories</h4>

Spark always keeps two releases in the mirror network: the most recent release on the current and
previous branches. To delete older versions simply use svn rm. The `downloads.js` file in the
website `js/` directory must also be updated to reflect the changes. For instance, the two
releases should be 1.1.1 and 1.0.2, but not 1.1.1 and 1.1.0.

```
$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
```

You should also delete the RC directories from the staging repository. For example:
After the vote passes and you moved the approved RC to the release repository, you should delete
the RC directories from the staging repository. For example:

```
svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
-m"Removing RC artifacts."
```

Make sure to also remove the unpublished staging repositories from the
<a href="https://repository.apache.org/">Apache Nexus Repository Manager</a>.


<h4>Remove Old Releases from Mirror Network</h4>

Spark always keeps the latest maintance released of each branch in the mirror network.
To delete older versions simply use svn rm:

```
$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
```

You will also need to update `js/download.js` to indicate the release is not mirrored
anymore, so that the correct links are generated on the site.

Also take a moment to check `HiveExternalCatalogVersionsSuite.scala` starting with branch-2.2
and see if it needs to be adjusted, since that test relies on mirrored downloads of previous
releases.


<h4>Update the Spark Apache Repository</h4>

Check out the tagged commit for the release candidate that passed and apply the correct version tag.
Expand Down
5 changes: 4 additions & 1 deletion site/doap.rdf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,13 +28,16 @@
<homepage rdf:resource="http://spark.apache.org" />
<asfext:pmc rdf:resource="http://spark.apache.org" />
<shortdesc>Apache Spark is a fast and general engine for large-scale data processing.</shortdesc>
<description>Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala and Python as well as a rich set of libraries including stream processing, machine learning, and graph analytics.</description>
<description>Apache Spark is a fast and general engine for large-scale data processing. It offers high-level APIs in Java, Scala, Python and R, as well as a rich set of libraries including stream processing, machine learning, and graph analytics.</description>
<bug-database rdf:resource="http://issues.apache.org/jira/browse/SPARK" />
<mailing-list rdf:resource="http://spark.apache.org/community.html" />
<download-page rdf:resource="http://spark.apache.org/downloads.html" />
<programming-language>Java</programming-language>
<programming-language>Scala</programming-language>
<programming-language>Python</programming-language>
<programming-language>R</programming-language>
<programming-language>SQL</programming-language>
<programming-language>JDBC</programming-language>
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍

<category rdf:resource="http://projects.apache.org/category/big-data" />
</Project>
</rdf:RDF>
54 changes: 26 additions & 28 deletions site/js/downloads.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,13 @@

releases = {};

function addRelease(version, releaseDate, packages, stable) {
releases[version] = {released: releaseDate, packages: packages, stable: stable};
function addRelease(version, releaseDate, packages, stable, mirrored) {
releases[version] = {
released: releaseDate,
packages: packages,
stable: stable,
mirrored: mirrored
};
}

var sources = {pretty: "Source Code", tag: "sources"};
Expand All @@ -24,26 +29,21 @@ var packagesV7 = [hadoop2p7, hadoop2p6, hadoop2p4, hadoop2p3, hadoopFree, source
// 2.2.0+
var packagesV8 = [hadoop2p7, hadoop2p6, hadoopFree, sources];

addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true);
addRelease("2.3.1", new Date("06/08/2018"), packagesV8, true, true);
addRelease("2.3.0", new Date("02/28/2018"), packagesV8, true, true);
addRelease("2.2.1", new Date("12/01/2017"), packagesV8, true, true);
addRelease("2.2.0", new Date("07/11/2017"), packagesV8, true, false);
addRelease("2.1.2", new Date("10/09/2017"), packagesV7, true, true);
addRelease("2.1.1", new Date("05/02/2017"), packagesV7, true, false);
addRelease("2.1.0", new Date("12/28/2016"), packagesV7, true, false);
addRelease("2.0.2", new Date("11/14/2016"), packagesV7, true, true);
addRelease("2.0.1", new Date("10/03/2016"), packagesV7, true, false);
addRelease("2.0.0", new Date("07/26/2016"), packagesV7, true, false);
//addRelease("2.0.0-preview", new Date("05/24/2016"), sources.concat(packagesV7), true, false);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true);
//addRelease("1.5.2", new Date("11/09/2015"), packagesV6, true);
//addRelease("1.5.1", new Date("10/02/2015"), packagesV6, true);
//addRelease("1.5.0", new Date("9/09/2015"), packagesV6, true);
//addRelease("1.4.1", new Date("7/15/2015"), packagesV6, true);
//addRelease("1.4.0", new Date("6/11/2015"), packagesV6, true);
addRelease("1.6.3", new Date("11/07/2016"), packagesV6, true, true);
addRelease("1.6.2", new Date("06/25/2016"), packagesV6, true, false);
addRelease("1.6.1", new Date("03/09/2016"), packagesV6, true, false);
addRelease("1.6.0", new Date("01/04/2016"), packagesV6, true, false);

function append(el, contents) {
el.innerHTML += contents;
Expand Down Expand Up @@ -128,10 +128,10 @@ function onVersionSelect() {
append(verifyLink, link);

// Populate releases
updateDownloadLink();
updateDownloadLink(releases[version].mirrored);
}

function updateDownloadLink() {
function updateDownloadLink(isMirrored) {
var versionSelect = document.getElementById("sparkVersionSelect");
var packageSelect = document.getElementById("sparkPackageSelect");
var downloadLink = document.getElementById("spanDownloadLink");
Expand All @@ -147,12 +147,10 @@ function updateDownloadLink() {
.replace(/-bin-sources/, ""); // special case for source packages

var link = "";
if (version < "1.6.3" ||
(version >= "2.0.0" && version <= "2.0.1") ||
(version >= "2.1.0" && version <= "2.1.1")) {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
} else {
if (isMirrored) {
link = "https://www.apache.org/dyn/closer.lua/spark/spark-$ver/$artifact";
} else {
link = "https://archive.apache.org/dist/spark/spark-$ver/$artifact";
}
link = link
.replace(/\$ver/, version)
Expand Down
30 changes: 20 additions & 10 deletions site/release-process.html
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -376,24 +376,34 @@ <h4>Publish to CRAN</h4>
<p>Publishing to CRAN is done using <a href="https://cran.r-project.org/submit.html">this form</a>.
Since it requires further manual steps, please also contact the <a href="mailto:private@spark.apache.org">PMC</a>.</p>

<h4>Remove Old Releases from Development Repository and Mirror Network</h4>
<h4> Remove RC artifacts from repositories</h4>

<p>Spark always keeps two releases in the mirror network: the most recent release on the current and
previous branches. To delete older versions simply use svn rm. The <code>downloads.js</code> file in the
website <code>js/</code> directory must also be updated to reflect the changes. For instance, the two
releases should be 1.1.1 and 1.0.2, but not 1.1.1 and 1.1.0.</p>

<pre><code>$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
</code></pre>

<p>You should also delete the RC directories from the staging repository. For example:</p>
<p>After the vote passes and you moved the approved RC to the release repository, you should delete
the RC directories from the staging repository. For example:</p>

<pre><code>svn rm https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-bin/ \
https://dist.apache.org/repos/dist/dev/spark/v2.3.1-rc1-docs/ \
-m"Removing RC artifacts."
</code></pre>

<p>Make sure to also remove the unpublished staging repositories from the
<a href="https://repository.apache.org/">Apache Nexus Repository Manager</a>.</p>

<h4>Remove Old Releases from Mirror Network</h4>

<p>Spark always keeps the latest maintance released of each branch in the mirror network.
To delete older versions simply use svn rm:</p>

<pre><code>$ svn rm https://dist.apache.org/repos/dist/release/spark/spark-1.1.0
</code></pre>

<p>You will also need to update <code>js/download.js</code> to indicate the release is not mirrored
anymore, so that the correct links are generated on the site.</p>

<p>Also take a moment to check <code>HiveExternalCatalogVersionsSuite.scala</code> starting with branch-2.2
and see if it needs to be adjusted, since that test relies on mirrored downloads of previous
releases.</p>

<h4>Update the Spark Apache Repository</h4>

<p>Check out the tagged commit for the release candidate that passed and apply the correct version tag.</p>
Expand Down