Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Release hash script and update hashes #230

Closed
wants to merge 9 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions gazebo/7/ubuntu/xenial/gzserver7/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
RUN . /etc/os-release \
&& echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -sc` main" > /etc/apt/sources.list.d/gazebo-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Fri, 16 Nov 2018 16:26:55 UTC" \
&& export SUM=c69ce7a69a8b65d88037a1b0aa39ce76ab6355787751afe725a79eda512e5e02 \
&& export FILE=/var/lib/apt/lists/packages.osrfoundation.org_gazebo_$ID-stable_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install gazebo packages
RUN apt-get update && apt-get install -q -y \
gazebo7=7.14.0-1* \
Expand Down
9 changes: 9 additions & 0 deletions gazebo/8/ubuntu/xenial/gzserver8/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
RUN . /etc/os-release \
&& echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -sc` main" > /etc/apt/sources.list.d/gazebo-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Fri, 16 Nov 2018 16:26:55 UTC" \
&& export SUM=c69ce7a69a8b65d88037a1b0aa39ce76ab6355787751afe725a79eda512e5e02 \
&& export FILE=/var/lib/apt/lists/packages.osrfoundation.org_gazebo_$ID-stable_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install gazebo packages
RUN apt-get update && apt-get install -q -y \
gazebo8=8.6.0-1* \
Expand Down
9 changes: 9 additions & 0 deletions gazebo/9/ubuntu/bionic/gzserver9/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
RUN . /etc/os-release \
&& echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -sc` main" > /etc/apt/sources.list.d/gazebo-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Mon, 05 Nov 2018 18:49:39 UTC" \
&& export SUM=ff31390e9da2b14d15f324620b11cde338224731c174612e84cc277d766d62a9 \
&& export FILE=/var/lib/apt/lists/packages.osrfoundation.org_gazebo_$ID-stable_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install gazebo packages
RUN apt-get update && apt-get install -q -y \
gazebo9=9.6.0-1* \
Expand Down
9 changes: 9 additions & 0 deletions gazebo/9/ubuntu/xenial/gzserver9/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys D2486D2DD8
RUN . /etc/os-release \
&& echo "deb http://packages.osrfoundation.org/gazebo/$ID-stable `lsb_release -sc` main" > /etc/apt/sources.list.d/gazebo-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Fri, 16 Nov 2018 16:26:55 UTC" \
&& export SUM=c69ce7a69a8b65d88037a1b0aa39ce76ab6355787751afe725a79eda512e5e02 \
&& export FILE=/var/lib/apt/lists/packages.osrfoundation.org_gazebo_$ID-stable_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install gazebo packages
RUN apt-get update && apt-get install -q -y \
gazebo9=9.6.0-1* \
Expand Down
146 changes: 146 additions & 0 deletions pr_docker_library.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,146 @@
#!/usr/bin/env python3

import datetime
import difflib
import git # gitpython
import github
import os
import shutil
import tempfile


# Path to root of this repo
BASE_PATH = os.path.dirname(os.path.abspath(__file__))
# Path to official docker library repo
DOCKER_LIBRARY_REPO_URL = 'git@github.com:docker-library/official-images.git'
DOCKER_LIBRARY_REPO = 'docker-library/official-images'
# Path to fork to update with docker images
FORK_LIBRARY_REPO_URL = 'git@github.com:ros-infrastructure/official-images.git'
FORK_LIBRARY_REPO_USER = 'ros-infrastructure'

# Token to use to interact with github
GIT_USER = os.environ.get('GITHUB_USER','')
GIT_EMAIL = os.environ.get('GITHUB_EMAIL','')
GIT_TOKEN = os.environ.get('GITHUB_TOKEN','')
GIT_AUTHOR = "{user} <{email}>".format(user=GIT_USER, email=GIT_EMAIL)
GIT_TOKEN = os.environ.get('GITHUB_TOKEN','')


PR_TEMPLATE = """
# Update library definition for '{name}'

<details><summary>Diff</summary>

```diff
{diff}
```
</details>
"""


def read_local_library_definition(name):
# Library definition file for `ros` is in `ros/ros`
with open(os.path.join(BASE_PATH, name, name), 'r') as fin:
return fin.read()


def read_upstream_library_definition(name):
# Get the manifest from the upstream repo
with tempfile.TemporaryDirectory(prefix='osrf-docker-images') as temp_dir:
git.Repo.clone_from(DOCKER_LIBRARY_REPO_URL, temp_dir, depth=1)
with open(os.path.join(temp_dir, 'library', name)) as fin:
return fin.read()


def pr_with_sentinel_exists(sentinel):
gh_handle = github.Github(GIT_TOKEN)
repo = gh_handle.get_repo(DOCKER_LIBRARY_REPO)
for pr in repo.get_pulls(state='open'):
if sentinel in pr.body:
return True
return False


def diff_library_definitions(local_ldf, upstream_ldf):
differ = difflib.Differ()
diff = differ.compare(
upstream_ldf.splitlines(keepends=True),
local_ldf.splitlines(keepends=True))
diff = ''.join(diff)
return diff


def update_library_definition_fork(name):
# Create branch on fork that is up to date with upstream
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this pushing the manifest (post merged) copied from osrf/docker_images to the FORK_LIBRARY_REPO_URL or only that from the checked out PR? To consolidate all the release changes for a given docker repo to one at a time, I think pushing the merged state of the repo manifest would be less overwhelming upstream that PRing each change. E.g. we'd only approve this automated upstream PR for the last merged PR to a given docker repo to batch release changes. This is similar to what we do now manually.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't understand the question. Here is an example PR created by this script: ros-infrastructure/official-images#8 . Does the diff answer it?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Your pr is a multiple merged distro changes, yet local PRs by the @osrf-docker-builder separate each distro PR, so the maintainer should be prudent in waiting for all the changes targeting a given repo manifest to be merged before upstreaming the final updated manifest. To avoid opening to many PR, they are batched into one periodic PR for that respective dockerhub repo.

Example PR:
#233
#234
#235
#236
#231

Upstream PR:
docker-library/official-images#5573

# write changed manifest to fork
# commit to new branch
# push branch to fork
branch_name = None
with tempfile.TemporaryDirectory(prefix='osrf-docker-images') as temp_dir:
# Clone fork repo locally with a remote for the official one
repo = git.Repo.clone_from(FORK_LIBRARY_REPO_URL, temp_dir, depth=1)
remote = repo.create_remote('upstream', DOCKER_LIBRARY_REPO_URL)
if not remote.exists():
raise RuntimeError('Failed to find official docker library')
remote.fetch()

# Create branch on fork repo tracking official docker library
d = datetime.datetime.utcnow()
d_str = d.strftime(r'%Y-%m-%d_%H-%M-%S')
branch_name = '{}/{}'.format(name, d_str)
head = repo.create_head(branch_name, remote.refs.master)
head.checkout()

# Update the library definition file on the fork
shutil.copy(
os.path.join(BASE_PATH, name, name),
os.path.join(temp_dir, 'library', name))

repo.git.add(all=True)
message = 'Update library definition for ' + name
repo.git.commit(message=message, author=GIT_AUTHOR)
repo.remotes.origin.push(branch_name, set_upstream=True)

print('created branch', branch_name)

return branch_name


def create_upstream_pull_request(name, diff, branch_name):
# open pull request with template
gh_handle = github.Github(GIT_TOKEN)
repo = gh_handle.get_repo(DOCKER_LIBRARY_REPO)
pr = repo.create_pull(
title='Update ' + name,
head='{}:{}'.format(FORK_LIBRARY_REPO_USER, branch_name),
base='master',
body=PR_TEMPLATE.format(name=name, diff=diff))
print('Created PR', pr.url)


def update_upstream_library_definition(name):
local_ldf = read_local_library_definition(name)
upstream_ldf = read_upstream_library_definition(name)

if local_ldf == upstream_ldf:
print('Upstream LDF and local LDF for {} are in sync'.format(name))
return

diff = diff_library_definitions(local_ldf, upstream_ldf)
print('--- Diff for {}---'.format(name))
print(diff)
print('--- End diff ---')

if pr_with_sentinel_exists(diff):
print('Upstream PR alredy created, ignoring')
return

fork_branch = update_library_definition_fork(name)

# Create pull request
create_upstream_pull_request(name, diff, fork_branch)


if __name__ == '__main__':

update_upstream_library_definition(os.environ['HUB_REPO'])
9 changes: 9 additions & 0 deletions ros/ardent/ubuntu/xenial/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,15 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
python3-vcstools \
&& rm -rf /var/lib/apt/lists/*

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Thu, 15 Nov 2018 21:55:38 UTC" \
&& export SUM=579ae6c6a96debdfb8300b998836ae23f3f6ea6609c0ab768af1b97885d4e7ce \
&& export FILE=/var/lib/apt/lists/repo.ros2.org_ubuntu_main_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# setup environment
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
Expand Down
9 changes: 9 additions & 0 deletions ros/bouncy/ubuntu/bionic/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,15 @@ RUN apt-get update && apt-get install --no-install-recommends -y \
python3-vcstools \
&& rm -rf /var/lib/apt/lists/*

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Thu, 15 Nov 2018 21:55:19 UTC" \
&& export SUM=125999ff7a89681dc97846b7adb233b342c469543c4a4832a22a6f82d4eb130b \
&& export FILE=/var/lib/apt/lists/repo.ros2.org_ubuntu_main_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# setup environment
ENV LANG C.UTF-8
ENV LC_ALL C.UTF-8
Expand Down
9 changes: 9 additions & 0 deletions ros/indigo/ubuntu/trusty/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
# setup sources.list
RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Tue, 13 Nov 2018 01:48:44 UTC" \
&& export SUM=b75a2d320917414042fd631bc44c6003414bf2c197c09e0ba73264633bc72345 \
&& export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install bootstrap tools
RUN apt-get update && apt-get install --no-install-recommends -y \
python-rosdep \
Expand Down
9 changes: 9 additions & 0 deletions ros/kinetic/ubuntu/xenial/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
# setup sources.list
RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Tue, 13 Nov 2018 04:34:23 UTC" \
&& export SUM=a243a54b0cea230b6e84d30ad38d8374f986ce38c166d38e6dbd6aaa0b38730e \
&& export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install bootstrap tools
RUN apt-get update && apt-get install --no-install-recommends -y \
python-rosdep \
Expand Down
9 changes: 9 additions & 0 deletions ros/lunar/debian/stretch/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
# setup sources.list
RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Wed, 07 Nov 2018 20:29:26 UTC" \
&& export SUM=019320d9c5c4095c0b44dbc1e6f695d9ebc583bafba9adfa1faed3a20c49d066 \
&& export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install bootstrap tools
RUN apt-get update && apt-get install --no-install-recommends -y \
python-rosdep \
Expand Down
9 changes: 9 additions & 0 deletions ros/lunar/ubuntu/xenial/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
# setup sources.list
RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Tue, 13 Nov 2018 04:34:23 UTC" \
&& export SUM=a243a54b0cea230b6e84d30ad38d8374f986ce38c166d38e6dbd6aaa0b38730e \
&& export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install bootstrap tools
RUN apt-get update && apt-get install --no-install-recommends -y \
python-rosdep \
Expand Down
9 changes: 9 additions & 0 deletions ros/melodic/debian/stretch/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
# setup sources.list
RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Wed, 07 Nov 2018 20:29:26 UTC" \
&& export SUM=019320d9c5c4095c0b44dbc1e6f695d9ebc583bafba9adfa1faed3a20c49d066 \
&& export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install bootstrap tools
RUN apt-get update && apt-get install --no-install-recommends -y \
python-rosdep \
Expand Down
9 changes: 9 additions & 0 deletions ros/melodic/ubuntu/bionic/ros-core/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,15 @@ RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 421C365BD9
# setup sources.list
RUN echo "deb http://packages.ros.org/ros/ubuntu `lsb_release -sc` main" > /etc/apt/sources.list.d/ros-latest.list

# break build cache for sync
RUN apt-get update \
&& . /etc/os-release \
&& echo "Release: Wed, 07 Nov 2018 20:28:21 UTC" \
&& export SUM=ceaf565b4dc80632569158783e148d7a8d05118cffb7ff34617f9ca18cca5bfc \
&& export FILE=/var/lib/apt/lists/packages.ros.org_ros_ubuntu_dists_$(lsb_release -sc)_InRelease \
&& echo "$SUM *$FILE" | sha256sum --check \
&& rm -rf /var/lib/apt/lists/*

# install bootstrap tools
RUN apt-get update && apt-get install --no-install-recommends -y \
python-rosdep \
Expand Down