From d0eae0ff5fcf3f4fc1f0ea6700a0510767e839dd Mon Sep 17 00:00:00 2001 From: AymenFJA Date: Tue, 1 Oct 2024 15:55:33 +0000 Subject: [PATCH] remove docker folder and link the tutorial repo in the README --- README.md | 4 + docker/README.md | 155 --------------------------------- docker/compose.yml | 29 ------ docker/radicalpilot.dockerfile | 123 -------------------------- docker/rp-complete.dockerfile | 107 ----------------------- 5 files changed, 4 insertions(+), 414 deletions(-) delete mode 100644 docker/README.md delete mode 100644 docker/compose.yml delete mode 100644 docker/radicalpilot.dockerfile delete mode 100644 docker/rp-complete.dockerfile diff --git a/README.md b/README.md index 34dcad543b..23ae386277 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,10 @@ RP documentation uses tutorials coded as Jupyter notebooks. `Sphinx` and the documentation also serves as a validation of your local development environment. +## Docker + +Please refer to the RADICAL-Cybertools [tutorial repository](https://github.com/radical-cybertools/tutorials) for instructions on how to build, use, and configure the Docker image of RP. + ## Provide Feedback Have a question, feature request or you found a bug? Feel free to open a diff --git a/docker/README.md b/docker/README.md deleted file mode 100644 index c12e3d163b..0000000000 --- a/docker/README.md +++ /dev/null @@ -1,155 +0,0 @@ -# Docker recipes - -Containerized RADICAL Pilot testing can be performed two ways. -Either the (required) MongoDB server can run in the same container -as RP, or in a separate service container. - -These recipes provide an environment with a password-less `rp` user -and a `/home/rp/rp-venv` Python virtual environment ready for use. - -## Monolithic container - -`rp-complete.dockerfile` provides a recipe for a complete container -to run MongoDB and the RP stack. Note that the MongoDB instance can -take a few seconds to start up. The easiest way to use the container -is to `run` the container's default mongod service, wait a few moments, -and then `exec` RP scripts or a shell in the container. - -When a container is launched from the resulting image with no arguments, -the entry point script will launch and initialize a MongoDB instance. -It will take a few seconds to be ready for connections, after which -radical.pilot will be able to use pymongo to connect (internally). - -When building the container, the local repository is used as the source -for the radical.pilot installation. The root of the repository is -referenced as the final argument to `docker build`. - -### Example - -From repository root directory - - docker build -t rp-complete -f docker/rp-complete.dockerfile . - -From the `docker` directory - - docker build -t rp-complete -f rp-complete.dockerfile .. - -Use the image to launch a container in the background with the name "rp_test". -Tell docker to remove the container after it exits. - - docker run --rm --name rp_test -d rp-complete - -Either use `-d` with `run`, or issue the `exec` in a separate terminal -after waiting a few seconds for the DB to be ready to accept connections. - - docker exec -ti -u rp rp_test bash -c \ - ". /home/rp/rp-venv/bin/activate && cd ~/radical.pilot && ~/rp-venv/bin/python -m pytest tests" - -The examples need the venv to be activated in order to find supporting -shell scripts on the default PATH. The current working directory also -needs to be writable. - - docker exec -ti -u rp rp_test bash -c \ - "cd && . /home/rp/rp-venv/bin/activate && python radical.pilot/examples/00*" - -If '-d' was used with 'run', you must kill or stop the container when done. - - docker stop rp_test - -## Docker Compose stack - -`compose.yml` provides a recipe for `docker compose` -(see https://docs.docker.com/compose/reference/). -The stack relies on two public container images -(`mongo:bionic` for the database service and -`mongo-express` for a database admin console) and a custom image -(to be built locally) providing the sshd service and login environment. -The custom image will be built automatically (with `docker compose --build`) -to provide a login environment with radical.pilot pre-installed and configured. -(See `radicalpilot.dockerfile` for details.) - -One service container is launched with each of these three images. -Services in the stack are named *mongo*, *mongo-express*, and *login*, respectively. -Use `docker compose` to run docker commands that reference the service name -instead of the container name. -(The actual container names are irrelevant.) - -The following examples are assumed to run from the `docker` directory in your -local copy of this repository. If run from a different directory, you will have -to give the path to the `compose.yml` file with the -[`-f` flag](https://docs.docker.com/engine/reference/commandline/compose/#use--f-to-specify-name-and-path-of-one-or-more-compose-files). - -### Building and launching services - -Bring the services up with - - docker compose up --build -d - -(See also https://docs.docker.com/engine/reference/commandline/compose_up/#options) - -### Using the container stack - -The service running the "radicalpilot" image (built from -radicalpilot.dockerfile in this directory) has an sshd server running and a -Python environment configured for the "rp" user. -Once the services are up, start a shell in the `loging` service container with - - docker compose exec -ti -u rp login bash - -or invoke a Radical Pilot test with, e.g. - - docker compose exec -ti -u rp login bash -c "cd && . /home/rp/rp-venv/bin/activate && python radical.pilot/examples/00*" - -### Port mapping services to the host machine - -Port 8081 is mapped from the *mongo-express* service container for the mongodb web interface. -To configure or disable, either edit compose.yml or use and explicit list of services to exclude. - - docker compose up -d login mongo - -Compose can also map the sshd port from the login service container to -port 2345 on the host machine loopback address (127.0.0.1). -Edit the compose.yml file to configure. - -### Don't forget to bring down the stack! - -Shut down and clean up with - - docker compose down - -## Development - -The local repository is copied into the image when it is built. -To update the radical.pilot software without rebuilding the image(s), -you can use `docker cp` or a `tar` pipeline. - -For a small number of files, for instance, - - docker cp ../examples/misc/raptor.cfg rp_test:/home/rp/radical.pilot/examples/misc/ - -To update the package from source, change directory to the root of the repository -or use `-C `. - -For monolithic `rp_test` container as described above: - - # From repo base - tar cf - . | docker cp - rp_test:/home/rp/radical.pilot/ - # From docker directory - tar cf - -C .. . | docker cp - rp_test:/home/rp/radical.pilot/ - -For `docker compose`: - - # From repo base - tar cf - . | docker compose -f docker/compose.yml cp - login:/home/rp/radical.pilot/ - # From docker directory - tar cf - -C .. . | docker compose cp - login:/home/rp/radical.pilot/ - -The `uid` in the container will not match your uid in the host, and the `cp` ran as root. -First, update the ownership and permissions of the transferred files. -Then reinstall the package in the container. - - docker exec -u root rp_test bash -c "chown -R rp /home/rp && chmod u+rx -R /home/rp/radical.pilot" - docker exec -u rp rp_test bash -c ". ~/rp-venv/bin/activate && cd ~/radical.pilot && pip install --upgrade ." - # or - docker compose exec -u root login bash -c "chown -R rp /home/rp && chmod u+rx -R /home/rp/radical.pilot" - docker compose exec -u rp login bash -c ". ~/rp-venv/bin/activate && cd ~/radical.pilot && pip install --upgrade ." diff --git a/docker/compose.yml b/docker/compose.yml deleted file mode 100644 index 23983b844b..0000000000 --- a/docker/compose.yml +++ /dev/null @@ -1,29 +0,0 @@ -# This file is hosted at https://github.com/radical-cybertools/radical.pilot/tree/devel/docker -# -# See README.md in this directory for instructions. -# -# See https://docs.docker.com/compose/compose-file/ for reference information. - -services: - mongo: - image: mongo:bionic - restart: unless-stopped - environment: - MONGO_INITDB_ROOT_USERNAME: root - MONGO_INITDB_ROOT_PASSWORD: password - mongo-express: - image: mongo-express - restart: unless-stopped - ports: - - "127.0.0.1:8081:8081" - environment: - ME_CONFIG_MONGODB_ADMINUSERNAME: root - ME_CONFIG_MONGODB_ADMINPASSWORD: password - login: - image: radicalpilot - restart: unless-stopped - build: - context: .. - dockerfile: docker/radicalpilot.dockerfile -# ports: -# - "127.0.0.1:2345:22" diff --git a/docker/radicalpilot.dockerfile b/docker/radicalpilot.dockerfile deleted file mode 100644 index ec78871a1d..0000000000 --- a/docker/radicalpilot.dockerfile +++ /dev/null @@ -1,123 +0,0 @@ -# This Dockerfile is used to create an image for the "radicalpilot" service in -# the compose.yml file in this directory. -# When a container is launched from this image with no arguments, the container -# will run an sshd daemon. -# Example: -# docker build -t radicalpilot -f radicalpilot.dockerfile .. -# -# This Dockerfile is hosted at https://github.com/radical-cybertools/radical.pilot/tree/devel/docker -# See README.md in this directory for instructions. -FROM ubuntu:focal - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get -yq --no-install-suggests --no-install-recommends install apt-utils build-essential software-properties-common && \ - apt-get install -y --no-install-recommends \ - curl \ - dnsutils \ - iputils-ping \ - language-pack-en \ - locales \ - && \ - rm -rf /var/lib/apt/lists/* - -RUN locale-gen en_US.UTF-8 && \ - update-locale LANG=en_US.UTF-8 - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y --no-install-recommends \ - gcc \ - git \ - libopenmpi-dev \ - openmpi-bin \ - openssh-server \ - vim \ - wget && \ - rm -rf /var/lib/apt/lists/* - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get -y --no-install-recommends install \ - libmpich-dev && \ - rm -rf /var/lib/apt/lists/* - -# Note that mpic++ can be configured with `update-alternatives` for openmpi or mpich. -# See https://stackoverflow.com/a/66538359/5351807 -# The mpic++ (both mpic++.openmpi and mpic++.mpich) uses g++ by default -# (and so is not affected by alternatives for "c++"). -# mpic++.openmpi can be redirected with environment variables (e.g. OMPI_CXX=clang++ mpic++ ...). -# It is not clear whether mpic++.mpich can be similarly configured, and it might be better -# to do fancy tool chain manipulation through Spack instead of the Ubuntu system tools. - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y \ - python3.8-dev \ - python3.9-dev \ - python3.8-venv \ - python3.9-venv \ - python-dev-is-python3 \ - tox && \ - rm -rf /var/lib/apt/lists/* - -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 8 -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.9 9 - -# Reference https://docs.docker.com/engine/examples/running_ssh_service/ -RUN mkdir /var/run/sshd - -# SSH login fix. Otherwise user is kicked off after login -RUN sed 's@session\s*required\s*pam_loginuid.so@session optional pam_loginuid.so@g' -i /etc/pam.d/sshd - -ENV NOTVISIBLE "in users profile" -RUN echo "export VISIBLE=now" >> /etc/profile - -EXPOSE 22 -CMD ["/usr/sbin/sshd", "-D"] - - -RUN groupadd radical && useradd -g radical -s /bin/bash -m rp -USER rp - -RUN (cd ~rp && python3 -m venv rp-venv) - -RUN (cd ~rp && \ - rp-venv/bin/pip install --upgrade --no-cache-dir \ - pip \ - setuptools \ - wheel && \ - rp-venv/bin/pip install --upgrade --no-cache-dir \ - coverage \ - flake8 \ - 'mock==2.0.0' \ - mpi4py \ - netifaces \ - ntplib \ - pylint \ - pymongo \ - pytest \ - python-hostlist \ - setproctitle \ - ) - -COPY --chown=rp:radical . /home/rp/radical.pilot - -RUN (. ~rp/rp-venv/bin/activate && \ - cd ~/radical.pilot && \ - ~rp/rp-venv/bin/pip install --no-cache-dir .) - - -USER root - -# Set the environment variable that Radical Pilot uses to find its MongoDB instance. -# Radical Pilot assumes the user is defined in the same database as in the URL. -# The Docker entry point creates users in the "admin" database, so we can just -# tell RP to use the same. The username and password are configured in the env -# passed to the mongo container in compose.yml. The service name from compose.yml -# also determines the URL host name. -# Note that the default mongodb port number is 27017. -ENV RADICAL_PILOT_DBURL="mongodb://root:password@mongo:27017/admin" - -RUN echo "export RADICAL_PILOT_DBURL=$RADICAL_PILOT_DBURL" >> /etc/profile diff --git a/docker/rp-complete.dockerfile b/docker/rp-complete.dockerfile deleted file mode 100644 index 1f5c6e8400..0000000000 --- a/docker/rp-complete.dockerfile +++ /dev/null @@ -1,107 +0,0 @@ -# This Dockerfile is hosted at https://github.com/radical-cybertools/radical.pilot/tree/devel/docker -# See README.md in this directory for instructions. - -FROM mongo:bionic -# Reference https://github.com/docker-library/mongo/blob/master/4.2/Dockerfile - -USER root - -RUN groupadd -r radical && useradd -r -g radical -m rp - -RUN apt-get update && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get -yq --no-install-suggests --no-install-recommends install apt-utils build-essential software-properties-common && \ - DEBIAN_FRONTEND=noninteractive \ - apt-get install -y --no-install-recommends \ - curl \ - dnsutils \ - gcc \ - git \ - iputils-ping \ - language-pack-en \ - libopenmpi-dev \ - locales \ - openmpi-bin \ - openssh-client \ - python3.8 \ - python3.8-dev \ - python3-venv \ - python3.8-venv \ - vim \ - wget && \ - rm -rf /var/lib/apt/lists/* - -RUN locale-gen en_US.UTF-8 && \ - update-locale LANG=en_US.UTF-8 - -RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 1 -RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 10 - -USER rp - -RUN (cd ~rp && python3.8 -m venv rp-venv) - -RUN (cd ~rp && \ - rp-venv/bin/pip install --upgrade --no-cache-dir \ - pip \ - setuptools \ - wheel && \ - rp-venv/bin/pip install --upgrade --no-cache-dir \ - coverage \ - flake8 \ - 'mock==2.0.0' \ - mpi4py \ - netifaces \ - ntplib \ - pylint \ - pymongo \ - pytest \ - python-hostlist \ - setproctitle \ - ) - -# Install RP from the current local repository working directory. -COPY --chown=rp:radical . /home/rp/radical.pilot -RUN . ~rp/rp-venv/bin/activate && \ - cd ~rp/radical.pilot && \ - pip install --no-cache-dir . - -# Note: if we want the image to target a specific (configrable) branch, use the following instead. -# -## Get repository for example and test files and to simplify RPREF build argument. -## Note that GitHub may have a source directory name suffix that does not exactly -## match the branch or tag name, so we use a glob to try to normalize the name. -#ARG RPREF="v1.5.2" -#RUN cd ~rp && \ -# wget https://github.com/radical-cybertools/radical.pilot/archive/$RPREF.tar.gz && \ -# tar zxvf $RPREF.tar.gz && \ -# mv radical.pilot-* radical.pilot && \ -# rm $RPREF.tar.gz -# -## Install RP from whichever git ref is provided as `--build-arg RPREF=...` (default 1.5.2) -#RUN . ~rp/rp-venv/bin/activate && \ -# cd ~rp/radical.pilot && \ -# pip install . -# OR -## Install official version from PyPI -#RUN . ~rp/rp-venv/bin/activate && \ -# pip install radical.pilot - -# Allow RADICAL Pilot to provide more useful behavior during testing, -# such as mocking missing resources from the resource specification. -ENV RADICAL_DEBUG="True" -RUN echo export RADICAL_DEBUG=$RADICAL_DEBUG >> ~rp/.profile - -USER root - -# Note that the following environment variables have special meaning to the -# `mongo` Docker container entry point script. -ENV MONGO_INITDB_ROOT_USERNAME=root -ENV MONGO_INITDB_ROOT_PASSWORD=password - -# Set the environment variable that Radical Pilot uses to find its MongoDB instance. -# Radical Pilot assumes the user is defined in the same database as in the URL. -# The Docker entry point creates users in the "admin" database, so we can just -# tell RP to use the same. -# Note that the default mongodb port number is 27017. -ENV RADICAL_PILOT_DBURL="mongodb://$MONGO_INITDB_ROOT_USERNAME:$MONGO_INITDB_ROOT_PASSWORD@localhost:27017/admin"