From a2c7e324731fadc5782f3b091eaf15d29cd01c72 Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Mon, 3 Jun 2024 13:11:19 +0100 Subject: [PATCH 01/11] main big reorg into single Dockerfile under env --- env/Dockerfile | 163 ++++++++++++++++++ {gds_dev => env/dev}/Dockerfile | 0 {gds_dev => env/dev}/README.md | 0 {gds_dev => env/dev}/check_dev_stack.ipynb | 0 {gds_dev => env/dev}/gds_dev.log | 0 {gds_dev => env/dev}/install_decktape.sh | 0 {gds_dev => env/dev}/install_gpq.sh | 0 {gds_dev => env/dev}/install_jekyll.sh | 0 {gds_dev => env/dev}/install_jupyter_dev.sh | 0 {gds_dev => env/dev}/install_latex_tools.sh | 0 {gds_dev => env/dev}/install_texbuild.py | 0 {gds_dev => env/dev}/texBuild.py | 0 {gds_dev => env/dev}/vimrc | 0 {gds_py => env/py}/Dockerfile | 0 {gds_py => env/py}/README.md | 0 {gds_py => env/py}/check_py_stack.ipynb | 0 {gds_py => env/py}/gds_py.yml | 0 {gds_py => env/py}/gds_py_build.log | 0 .../py}/gds_py_explicit_macos-arm.txt | 0 .../py}/gds_py_explicit_macos-latest.txt | 0 .../py}/gds_py_explicit_ubuntu-latest.txt | 0 .../py}/gds_py_explicit_windows-latest.txt | 0 {gds_py => env/py}/gds_py_win_installer.bat | 0 {gds_py => env/py}/stack_py.md | 0 {gds_py => env/py}/stack_py.txt | 0 {gds_py => env/py}/test_py_stack.ipynb | 0 {gds => env/r}/Dockerfile | 0 {gds => env/r}/README.md | 0 {gds => env/r}/check_r_stack.ipynb | 0 {gds => env/r}/gds_build.log | 0 {gds => env/r}/gds_latest.log | 0 {gds => env/r}/install_R.sh | 0 {gds => env/r}/install_R_gds.sh | 0 {gds => env/r}/install_R_stack.sh | 0 {gds => env/r}/install_quarto.sh | 0 {gds => env/r}/install_tippecanoe.sh | 0 {gds => env/r}/setup_py-r.sh | 0 {gds => env/r}/stack_r.md | 0 {gds => env/r}/stack_r.txt | 0 {gds => env/r}/test_courses.ipynb | 0 {gds_code => frontend_code}/Dockerfile | 0 41 files changed, 163 insertions(+) create mode 100644 env/Dockerfile rename {gds_dev => env/dev}/Dockerfile (100%) rename {gds_dev => env/dev}/README.md (100%) rename {gds_dev => env/dev}/check_dev_stack.ipynb (100%) rename {gds_dev => env/dev}/gds_dev.log (100%) rename {gds_dev => env/dev}/install_decktape.sh (100%) rename {gds_dev => env/dev}/install_gpq.sh (100%) rename {gds_dev => env/dev}/install_jekyll.sh (100%) rename {gds_dev => env/dev}/install_jupyter_dev.sh (100%) rename {gds_dev => env/dev}/install_latex_tools.sh (100%) rename {gds_dev => env/dev}/install_texbuild.py (100%) rename {gds_dev => env/dev}/texBuild.py (100%) rename {gds_dev => env/dev}/vimrc (100%) rename {gds_py => env/py}/Dockerfile (100%) rename {gds_py => env/py}/README.md (100%) rename {gds_py => env/py}/check_py_stack.ipynb (100%) rename {gds_py => env/py}/gds_py.yml (100%) rename {gds_py => env/py}/gds_py_build.log (100%) rename {gds_py => env/py}/gds_py_explicit_macos-arm.txt (100%) rename {gds_py => env/py}/gds_py_explicit_macos-latest.txt (100%) rename {gds_py => env/py}/gds_py_explicit_ubuntu-latest.txt (100%) rename {gds_py => env/py}/gds_py_explicit_windows-latest.txt (100%) rename {gds_py => env/py}/gds_py_win_installer.bat (100%) rename {gds_py => env/py}/stack_py.md (100%) rename {gds_py => env/py}/stack_py.txt (100%) rename {gds_py => env/py}/test_py_stack.ipynb (100%) rename {gds => env/r}/Dockerfile (100%) rename {gds => env/r}/README.md (100%) rename {gds => env/r}/check_r_stack.ipynb (100%) rename {gds => env/r}/gds_build.log (100%) rename {gds => env/r}/gds_latest.log (100%) rename {gds => env/r}/install_R.sh (100%) rename {gds => env/r}/install_R_gds.sh (100%) rename {gds => env/r}/install_R_stack.sh (100%) rename {gds => env/r}/install_quarto.sh (100%) rename {gds => env/r}/install_tippecanoe.sh (100%) rename {gds => env/r}/setup_py-r.sh (100%) rename {gds => env/r}/stack_r.md (100%) rename {gds => env/r}/stack_r.txt (100%) rename {gds => env/r}/test_courses.ipynb (100%) rename {gds_code => frontend_code}/Dockerfile (100%) diff --git a/env/Dockerfile b/env/Dockerfile new file mode 100644 index 0000000..09b21a5 --- /dev/null +++ b/env/Dockerfile @@ -0,0 +1,163 @@ +FROM quay.io/jupyter/minimal-notebook + +LABEL maintainer="Dani Arribas-Bel " + +# Set version +ENV GDS_ENV_VERSION "11.0alpha" + +# https://github.com/ContinuumIO/docker-images/blob/master/miniconda3/Dockerfile +ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 + +ADD ./*/*.sh $HOME/scripts/ +USER root +RUN chmod +x $HOME/scripts/*.sh +USER $NB_UID + + ############## + ### Python ### + ############## +#--- Python ---# + +ADD ./py/gds_py.yml /home/${NB_USER}/ +RUN mamba env create -f gds_py.yml \ + && source activate gds \ + && python -m ipykernel install --user --name gds --display-name "GDS-$GDS_ENV_VERSION" \ + && conda deactivate \ + && rm ./gds_py.yml \ + && conda clean --all --yes --force-pkgs-dirs \ + && find /opt/conda/ -follow -type f -name '*.a' -delete \ + && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ + && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ + && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ + -follow -type f -name '*.js' ! -name '*.min.js' -delete \ + && pip cache purge \ + && rm -rf /home/$NB_USER/.cache/pip + +# Make GDS default +RUN jupyter lab --generate-config \ + && echo "c.MultiKernelManager.default_kernel_name='gds'" >> \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ + && echo "conda activate gds" >> /home/${NB_USER}/.bashrc \ + && echo "c.KernelSpecManager.ensure_native_kernel = False" >> \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ +# https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982 + && echo "c.KernelSpecManager.whitelist = {'gds'}" >> \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py +RUN jupyter kernelspec remove -y python3 +ENV PATH="/opt/conda/envs/gds/bin/:${PATH}" + +#--- Jupyter config ---# +USER root +# Turn off notifications +RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \ +# Clean cache up + && jupyter lab clean -y \ + && conda clean --all -f -y \ + && npm cache clean --force \ + && rm -rf $CONDA_DIR/share/jupyter/lab/staging \ + && rm -rf "/home/${NB_USER}/.node-gyp" \ + && rm -rf /home/$NB_USER/.cache/yarn \ +# Fix permissions + && fix-permissions "${CONDA_DIR}" \ + && fix-permissions "/home/${NB_USER}" +# Build mpl font cache +# https://github.com/jupyter/docker-stacks/blob/c3d5df67c8b158b0aded401a647ea97ada1dd085/scipy-notebook/Dockerfile#L59 +USER $NB_UID +ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" +RUN source activate gds \ + && MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ + fix-permissions "/home/${NB_USER}" + + ######### + ### R ### + ######### + +USER root + +# Remove Conda from path to not interfere with R install +RUN echo ${PATH} +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUN echo ${PATH} + +# install R +RUN $HOME/scripts/install_R.sh + +# Install Tidyverse + Geospatial +RUN $HOME/scripts/install_R_stack.sh + +# install R GDS stack +RUN $HOME/scripts/install_R_gds.sh + +# re-attach conda to path +ENV PATH="/opt/conda/envs/gds/bin/:/opt/conda/condabin:/opt/conda/envs/gds/bin:/opt/conda/bin:${PATH}" + +#--- R/Python ---# + +RUN $HOME/scripts/setup_py-r.sh + +#--- Quarto ---# +RUN $HOME/scripts/install_quarto.sh + + ########### + ### Dev ### + ########### + +USER $NB_UID + +#--- Jupyter Dev ---# +RUN $HOME/scripts/install_jupyter_dev.sh + +#--- jekyll ---# +RUN $HOME/scripts/install_jekyll.sh + + +USER root + +#--- tippecanoe ---# +RUN $HOME/scripts/install_tippecanoe.sh + +#--- htop ---# +RUN apt-get update \ + && apt-get install -y --no-install-recommends htop + +#--- Decktape ---# +RUN $HOME/scripts/install_decktape.sh + +#--- LaTeX tools ---# +ADD ./dev/texBuild.py $HOME/ +ADD ./dev/install_texbuild.py $HOME/ +RUN $HOME/scripts/install_latex_tools.sh + +#--- Vim ---# +ADD ./dev/vimrc $HOME/.vimrc +RUN apt-get update \ + && apt-get install -y vim \ + && curl -fLo $HOME/.vim/autoload/plug.vim --create-dirs \ + https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim \ + && vim +PlugInstall +qall + +# GPQ +RUN $HOME/scripts/install_gpq.sh + +#--- Clean up ---# +RUN find /opt/conda/ -follow -type f -name '*.a' -delete \ + && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ + && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ + && rm -rf /home/$NB_USER/.cache/pip + +RUN cd $NB_HOME \ + && conda clean --yes --all \ + && rm -rf /home/$NB_USER/.cache/pip \ + && rm -rf /tmp/downloaded_packages/ /tmp/*.rds \ + && rm -rf $GEM_HOME/cache \ + && rm -rf /usr/local/bundle/cache \ + && apt-get autoclean \ + && apt-get autoremove \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* \ + && rm -rf $HOME/scripts +RUN fix-permissions $HOME \ + && fix-permissions $CONDA_DIR + +# Switch back to jovyan to avoid accidental container runs as root +USER $NB_UID diff --git a/gds_dev/Dockerfile b/env/dev/Dockerfile similarity index 100% rename from gds_dev/Dockerfile rename to env/dev/Dockerfile diff --git a/gds_dev/README.md b/env/dev/README.md similarity index 100% rename from gds_dev/README.md rename to env/dev/README.md diff --git a/gds_dev/check_dev_stack.ipynb b/env/dev/check_dev_stack.ipynb similarity index 100% rename from gds_dev/check_dev_stack.ipynb rename to env/dev/check_dev_stack.ipynb diff --git a/gds_dev/gds_dev.log b/env/dev/gds_dev.log similarity index 100% rename from gds_dev/gds_dev.log rename to env/dev/gds_dev.log diff --git a/gds_dev/install_decktape.sh b/env/dev/install_decktape.sh similarity index 100% rename from gds_dev/install_decktape.sh rename to env/dev/install_decktape.sh diff --git a/gds_dev/install_gpq.sh b/env/dev/install_gpq.sh similarity index 100% rename from gds_dev/install_gpq.sh rename to env/dev/install_gpq.sh diff --git a/gds_dev/install_jekyll.sh b/env/dev/install_jekyll.sh similarity index 100% rename from gds_dev/install_jekyll.sh rename to env/dev/install_jekyll.sh diff --git a/gds_dev/install_jupyter_dev.sh b/env/dev/install_jupyter_dev.sh similarity index 100% rename from gds_dev/install_jupyter_dev.sh rename to env/dev/install_jupyter_dev.sh diff --git a/gds_dev/install_latex_tools.sh b/env/dev/install_latex_tools.sh similarity index 100% rename from gds_dev/install_latex_tools.sh rename to env/dev/install_latex_tools.sh diff --git a/gds_dev/install_texbuild.py b/env/dev/install_texbuild.py similarity index 100% rename from gds_dev/install_texbuild.py rename to env/dev/install_texbuild.py diff --git a/gds_dev/texBuild.py b/env/dev/texBuild.py similarity index 100% rename from gds_dev/texBuild.py rename to env/dev/texBuild.py diff --git a/gds_dev/vimrc b/env/dev/vimrc similarity index 100% rename from gds_dev/vimrc rename to env/dev/vimrc diff --git a/gds_py/Dockerfile b/env/py/Dockerfile similarity index 100% rename from gds_py/Dockerfile rename to env/py/Dockerfile diff --git a/gds_py/README.md b/env/py/README.md similarity index 100% rename from gds_py/README.md rename to env/py/README.md diff --git a/gds_py/check_py_stack.ipynb b/env/py/check_py_stack.ipynb similarity index 100% rename from gds_py/check_py_stack.ipynb rename to env/py/check_py_stack.ipynb diff --git a/gds_py/gds_py.yml b/env/py/gds_py.yml similarity index 100% rename from gds_py/gds_py.yml rename to env/py/gds_py.yml diff --git a/gds_py/gds_py_build.log b/env/py/gds_py_build.log similarity index 100% rename from gds_py/gds_py_build.log rename to env/py/gds_py_build.log diff --git a/gds_py/gds_py_explicit_macos-arm.txt b/env/py/gds_py_explicit_macos-arm.txt similarity index 100% rename from gds_py/gds_py_explicit_macos-arm.txt rename to env/py/gds_py_explicit_macos-arm.txt diff --git a/gds_py/gds_py_explicit_macos-latest.txt b/env/py/gds_py_explicit_macos-latest.txt similarity index 100% rename from gds_py/gds_py_explicit_macos-latest.txt rename to env/py/gds_py_explicit_macos-latest.txt diff --git a/gds_py/gds_py_explicit_ubuntu-latest.txt b/env/py/gds_py_explicit_ubuntu-latest.txt similarity index 100% rename from gds_py/gds_py_explicit_ubuntu-latest.txt rename to env/py/gds_py_explicit_ubuntu-latest.txt diff --git a/gds_py/gds_py_explicit_windows-latest.txt b/env/py/gds_py_explicit_windows-latest.txt similarity index 100% rename from gds_py/gds_py_explicit_windows-latest.txt rename to env/py/gds_py_explicit_windows-latest.txt diff --git a/gds_py/gds_py_win_installer.bat b/env/py/gds_py_win_installer.bat similarity index 100% rename from gds_py/gds_py_win_installer.bat rename to env/py/gds_py_win_installer.bat diff --git a/gds_py/stack_py.md b/env/py/stack_py.md similarity index 100% rename from gds_py/stack_py.md rename to env/py/stack_py.md diff --git a/gds_py/stack_py.txt b/env/py/stack_py.txt similarity index 100% rename from gds_py/stack_py.txt rename to env/py/stack_py.txt diff --git a/gds_py/test_py_stack.ipynb b/env/py/test_py_stack.ipynb similarity index 100% rename from gds_py/test_py_stack.ipynb rename to env/py/test_py_stack.ipynb diff --git a/gds/Dockerfile b/env/r/Dockerfile similarity index 100% rename from gds/Dockerfile rename to env/r/Dockerfile diff --git a/gds/README.md b/env/r/README.md similarity index 100% rename from gds/README.md rename to env/r/README.md diff --git a/gds/check_r_stack.ipynb b/env/r/check_r_stack.ipynb similarity index 100% rename from gds/check_r_stack.ipynb rename to env/r/check_r_stack.ipynb diff --git a/gds/gds_build.log b/env/r/gds_build.log similarity index 100% rename from gds/gds_build.log rename to env/r/gds_build.log diff --git a/gds/gds_latest.log b/env/r/gds_latest.log similarity index 100% rename from gds/gds_latest.log rename to env/r/gds_latest.log diff --git a/gds/install_R.sh b/env/r/install_R.sh similarity index 100% rename from gds/install_R.sh rename to env/r/install_R.sh diff --git a/gds/install_R_gds.sh b/env/r/install_R_gds.sh similarity index 100% rename from gds/install_R_gds.sh rename to env/r/install_R_gds.sh diff --git a/gds/install_R_stack.sh b/env/r/install_R_stack.sh similarity index 100% rename from gds/install_R_stack.sh rename to env/r/install_R_stack.sh diff --git a/gds/install_quarto.sh b/env/r/install_quarto.sh similarity index 100% rename from gds/install_quarto.sh rename to env/r/install_quarto.sh diff --git a/gds/install_tippecanoe.sh b/env/r/install_tippecanoe.sh similarity index 100% rename from gds/install_tippecanoe.sh rename to env/r/install_tippecanoe.sh diff --git a/gds/setup_py-r.sh b/env/r/setup_py-r.sh similarity index 100% rename from gds/setup_py-r.sh rename to env/r/setup_py-r.sh diff --git a/gds/stack_r.md b/env/r/stack_r.md similarity index 100% rename from gds/stack_r.md rename to env/r/stack_r.md diff --git a/gds/stack_r.txt b/env/r/stack_r.txt similarity index 100% rename from gds/stack_r.txt rename to env/r/stack_r.txt diff --git a/gds/test_courses.ipynb b/env/r/test_courses.ipynb similarity index 100% rename from gds/test_courses.ipynb rename to env/r/test_courses.ipynb diff --git a/gds_code/Dockerfile b/frontend_code/Dockerfile similarity index 100% rename from gds_code/Dockerfile rename to frontend_code/Dockerfile From df4ba5f271006cf30d0402e733e20a9cfde7544b Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Mon, 3 Jun 2024 22:43:29 +0100 Subject: [PATCH 02/11] Clean up main Dockerfile from redundant commands --- env/Dockerfile | 24 +++++++----------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/env/Dockerfile b/env/Dockerfile index 09b21a5..f08d9f8 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -23,15 +23,7 @@ RUN mamba env create -f gds_py.yml \ && source activate gds \ && python -m ipykernel install --user --name gds --display-name "GDS-$GDS_ENV_VERSION" \ && conda deactivate \ - && rm ./gds_py.yml \ - && conda clean --all --yes --force-pkgs-dirs \ - && find /opt/conda/ -follow -type f -name '*.a' -delete \ - && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ - && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ - && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ - -follow -type f -name '*.js' ! -name '*.min.js' -delete \ - && pip cache purge \ - && rm -rf /home/$NB_USER/.cache/pip + && rm ./gds_py.yml # Make GDS default RUN jupyter lab --generate-config \ @@ -51,22 +43,16 @@ USER root # Turn off notifications RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \ # Clean cache up - && jupyter lab clean -y \ - && conda clean --all -f -y \ && npm cache clean --force \ && rm -rf $CONDA_DIR/share/jupyter/lab/staging \ && rm -rf "/home/${NB_USER}/.node-gyp" \ && rm -rf /home/$NB_USER/.cache/yarn \ -# Fix permissions - && fix-permissions "${CONDA_DIR}" \ - && fix-permissions "/home/${NB_USER}" # Build mpl font cache # https://github.com/jupyter/docker-stacks/blob/c3d5df67c8b158b0aded401a647ea97ada1dd085/scipy-notebook/Dockerfile#L59 USER $NB_UID ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" RUN source activate gds \ - && MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ - fix-permissions "/home/${NB_USER}" + && MPLBACKEND=Agg python -c "import matplotlib.pyplot" ######### ### R ### @@ -143,10 +129,14 @@ RUN $HOME/scripts/install_gpq.sh RUN find /opt/conda/ -follow -type f -name '*.a' -delete \ && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ + && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ + -follow -type f -name '*.js' ! -name '*.min.js' -delete \ + && pip cache purge \ && rm -rf /home/$NB_USER/.cache/pip RUN cd $NB_HOME \ - && conda clean --yes --all \ + && jupyter lab clean -y \ + && conda clean --yes --all --force-pkgs-dirs \ && rm -rf /home/$NB_USER/.cache/pip \ && rm -rf /tmp/downloaded_packages/ /tmp/*.rds \ && rm -rf $GEM_HOME/cache \ From 3dcaaeaf55de6e5e1c7c03ac7354c0866fbb47bb Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Mon, 3 Jun 2024 22:44:10 +0100 Subject: [PATCH 03/11] Update make tests and stacks --- Makefile | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 29def35..13fb3f1 100644 --- a/Makefile +++ b/Makefile @@ -1,24 +1,25 @@ export GDS_VERSION=10.0 +# make command [image=image_name] #DOCKERRUN = docker run --rm --user root -e GRANT_SUDO=yes -e NB_UID=1002 -e NB_GID=100 -v `pwd`:/home/jovyan/test DOCKERRUN = docker run -v `pwd`:/home/jovyan/test test: test_py test_r test_py: - $(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh jupyter nbconvert --to html --execute /home/jovyan/test/gds_py/check_py_stack.ipynb + $(DOCKERRUN) $(image) start.sh /opt/conda/envs/gds/bin/jupyter nbconvert --to html --execute /home/jovyan/test/env/py/check_py_stack.ipynb test_r: - $(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh jupyter nbconvert --to html --execute /home/jovyan/test/gds/check_r_stack.ipynb + $(DOCKERRUN) $(image) start.sh /opt/conda/envs/gds/bin/jupyter nbconvert --to html --execute /home/jovyan/test/env/r/check_r_stack.ipynb write_stacks: write_py_stack write_r_stack write_py_stack: - $(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh bash -c "conda list -n gds > /home/jovyan/test/gds_py/stack_py.txt" - $(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh sed -i '1iGDS version: ${GDS_VERSION}' /home/jovyan/test/gds_py/stack_py.txt - $(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh python -c "import subprocess, pandas; fo=open('/home/jovyan/test/gds_py/stack_py.md', 'w'); fo.write(pandas.read_json(subprocess.check_output(['conda', 'list', '-n', 'gds', '--json']).decode())[['name', 'version', 'build_string', 'channel']].to_markdown());fo.close()" - $(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh sed -i "1s/^/\n/" /home/jovyan/test/gds_py/stack_py.md + $(DOCKERRUN) $(image) start.sh bash -c "conda list -n gds > /home/jovyan/test/env/py/stack_py.txt" + $(DOCKERRUN) $(image) start.sh sed -i '1iGDS image: $(image)' /home/jovyan/test/env/py/stack_py.txt + $(DOCKERRUN) $(image) start.sh /opt/conda/envs/gds/bin/python -c "import subprocess, pandas; fo=open('/home/jovyan/test/env/py/stack_py.md', 'w'); fo.write(pandas.read_json(subprocess.check_output(['conda', 'list', '-n', 'gds', '--json']).decode())[['name', 'version', 'build_string', 'channel']].to_markdown());fo.close()" + $(DOCKERRUN) $(image) start.sh sed -i "1s/^/\n/" /home/jovyan/test/env/py/stack_py.md write_r_stack: - $(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh sh -c 'Rscript -e "ip <- as.data.frame(installed.packages()[,c(1,3:4)]); print(ip)" > /home/jovyan/test/gds/stack_r.txt' - $(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh sed -i '1iGDS version: ${GDS_VERSION}' /home/jovyan/test/gds/stack_r.txt - $(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh Rscript -e "library(knitr); ip <- as.data.frame(installed.packages()[,c(1,3:4)]); fc <- file('/home/jovyan/test/gds/stack_r.md'); writeLines(kable(ip, format = 'markdown'), fc); close(fc);" - $(DOCKERRUN) darribas/gds:${GDS_VERSION} start.sh sed -i "1s/^/\n/" /home/jovyan/test/gds/stack_r.md + $(DOCKERRUN) $(image) start.sh sh -c 'Rscript -e "ip <- as.data.frame(installed.packages()[,c(1,3:4)]); print(ip)" > /home/jovyan/test/env/r/stack_r.txt' + $(DOCKERRUN) $(image) start.sh sed -i '1iGDS image: $(image)' /home/jovyan/test/env/r/stack_r.txt + $(DOCKERRUN) $(image) start.sh Rscript -e "library(knitr); ip <- as.data.frame(installed.packages()[,c(1,3:4)]); fc <- file('/home/jovyan/test/env/r/stack_r.md'); writeLines(kable(ip, format = 'markdown'), fc); close(fc);" + $(DOCKERRUN) $(image) start.sh sed -i "1s/^/\n/" /home/jovyan/test/env/r/stack_r.md write_py_explicit: - $(DOCKERRUN) darribas/gds_py:${GDS_VERSION} start.sh sh -c "conda list -n gds --explicit > /home/jovyan/test/gds_py/stack_py_explicit.txt" + $(DOCKERRUN) $(image) start.sh sh -c "conda list -n gds --explicit > /home/jovyan/test/env/py/stack_py_explicit.txt" website_build: cd website && \ rm -rf _includes && \ From abf52c887a4d3870ae80825e777546620cee8e6a Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Wed, 5 Jun 2024 23:04:35 +0100 Subject: [PATCH 04/11] Add jekyll install cross-arch + move vim install to script --- env/Dockerfile | 12 ++++-------- env/dev/install_jekyll.sh | 20 ++++++-------------- env/dev/install_jupyter_dev.sh | 2 ++ env/dev/install_vim.sh | 12 ++++++++++++ 4 files changed, 24 insertions(+), 22 deletions(-) create mode 100644 env/dev/install_vim.sh diff --git a/env/Dockerfile b/env/Dockerfile index f08d9f8..25fe3d0 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -93,12 +93,12 @@ USER $NB_UID #--- Jupyter Dev ---# RUN $HOME/scripts/install_jupyter_dev.sh -#--- jekyll ---# -RUN $HOME/scripts/install_jekyll.sh - USER root +#--- jekyll ---# +RUN $HOME/scripts/install_jekyll.sh + #--- tippecanoe ---# RUN $HOME/scripts/install_tippecanoe.sh @@ -116,11 +116,7 @@ RUN $HOME/scripts/install_latex_tools.sh #--- Vim ---# ADD ./dev/vimrc $HOME/.vimrc -RUN apt-get update \ - && apt-get install -y vim \ - && curl -fLo $HOME/.vim/autoload/plug.vim --create-dirs \ - https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim \ - && vim +PlugInstall +qall +RUN $HOME/scripts/install_vim.sh # GPQ RUN $HOME/scripts/install_gpq.sh diff --git a/env/dev/install_jekyll.sh b/env/dev/install_jekyll.sh index 6c7d5b6..60806e6 100644 --- a/env/dev/install_jekyll.sh +++ b/env/dev/install_jekyll.sh @@ -1,17 +1,9 @@ #!/bin/bash -#--- [Run under $NB_USER] ---# +apt-get update \ + && apt-get install -y ruby-full build-essential zlib1g-dev +# https://github.com/sass-contrib/sass-embedded-host-ruby/issues/130#issuecomment-1588245011 +gem install sass-embedded +gem install sass --force sass-embedded +gem install jekyll bundler github-pages jekyll-scholar just-the-docs -# Update w/ https://github.com/github/pages-gem/blob/master/Dockerfile - -if [ "$(uname -m)" = "x86_64" ]; then - mamba create -n dev --yes -c conda-forge \ - gcc \ - rb-bundler \ - && source activate dev \ - && gem install github-pages jekyll-scholar just-the-docs \ - && source activate base \ - && pyppeteer-install -else - echo "Jekyll with Github Pages is only supported on x86_64 architectures" -fi diff --git a/env/dev/install_jupyter_dev.sh b/env/dev/install_jupyter_dev.sh index 48912e0..59149ea 100644 --- a/env/dev/install_jupyter_dev.sh +++ b/env/dev/install_jupyter_dev.sh @@ -18,4 +18,6 @@ pip install dask-labextension bash_kernel jupyterlab_vim \ sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'ir'}/c.KernelSpecManager.whitelist = {'gds', 'ir', 'bash'}/g" \ /home/${NB_USER}/.jupyter/jupyter_lab_config.py +#--- Pypeteer ---# +pyppeteer-install diff --git a/env/dev/install_vim.sh b/env/dev/install_vim.sh new file mode 100644 index 0000000..1fc248d --- /dev/null +++ b/env/dev/install_vim.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +################# +### Vim setup ### +################# + +apt-get update \ + && apt-get install -y vim \ + && curl -fLo $HOME/.vim/autoload/plug.vim --create-dirs \ + https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim \ + && vim +PlugInstall +qall + From 320e431254ee01aae49ed1492e7f9219699047e5 Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Fri, 7 Jun 2024 08:00:09 +0100 Subject: [PATCH 05/11] Add back cleaning statements throughout to keep overall size in check --- env/Dockerfile | 43 ++++++++++++++++++++++--------------- env/dev/install_decktape.sh | 1 + env/dev/install_jekyll.sh | 5 ++++- env/dev/install_vim.sh | 8 +++++-- env/r/install_R.sh | 6 ++++-- env/r/install_R_gds.sh | 15 ------------- env/r/setup_py-r.sh | 2 ++ 7 files changed, 43 insertions(+), 37 deletions(-) diff --git a/env/Dockerfile b/env/Dockerfile index 25fe3d0..9742653 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -23,7 +23,15 @@ RUN mamba env create -f gds_py.yml \ && source activate gds \ && python -m ipykernel install --user --name gds --display-name "GDS-$GDS_ENV_VERSION" \ && conda deactivate \ - && rm ./gds_py.yml + && rm ./gds_py.yml \ + && conda clean --yes --all --force-pkgs-dirs \ + && find /opt/conda/ -follow -type f -name '*.a' -delete \ + && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ + && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ + && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ + -follow -type f -name '*.js' ! -name '*.min.js' -delete \ + && pip cache purge \ + && rm -rf $HOME/.cache/pip # Make GDS default RUN jupyter lab --generate-config \ @@ -38,15 +46,24 @@ RUN jupyter lab --generate-config \ RUN jupyter kernelspec remove -y python3 ENV PATH="/opt/conda/envs/gds/bin/:${PATH}" +#--- Jupyter Dev ---# +RUN $HOME/scripts/install_jupyter_dev.sh + #--- Jupyter config ---# + USER root + # Turn off notifications RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \ # Clean cache up + && jupyter lab clean -y \ + && conda clean --yes --all --force-pkgs-dirs \ && npm cache clean --force \ && rm -rf $CONDA_DIR/share/jupyter/lab/staging \ && rm -rf "/home/${NB_USER}/.node-gyp" \ && rm -rf /home/$NB_USER/.cache/yarn \ + && fix-permissions "${CONDA_DIR}" \ + && fix-permissions "/home/${NB_USER}" # Build mpl font cache # https://github.com/jupyter/docker-stacks/blob/c3d5df67c8b158b0aded401a647ea97ada1dd085/scipy-notebook/Dockerfile#L59 USER $NB_UID @@ -88,14 +105,6 @@ RUN $HOME/scripts/install_quarto.sh ### Dev ### ########### -USER $NB_UID - -#--- Jupyter Dev ---# -RUN $HOME/scripts/install_jupyter_dev.sh - - -USER root - #--- jekyll ---# RUN $HOME/scripts/install_jekyll.sh @@ -118,22 +127,25 @@ RUN $HOME/scripts/install_latex_tools.sh ADD ./dev/vimrc $HOME/.vimrc RUN $HOME/scripts/install_vim.sh -# GPQ +#--- GPQ ---# RUN $HOME/scripts/install_gpq.sh #--- Clean up ---# +RUN fix-permissions $HOME \ + && fix-permissions $CONDA_DIR + +USER $NB_UID + RUN find /opt/conda/ -follow -type f -name '*.a' -delete \ && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ -follow -type f -name '*.js' ! -name '*.min.js' -delete \ && pip cache purge \ - && rm -rf /home/$NB_USER/.cache/pip + && rm -rf $HOME/.cache/pip +USER root RUN cd $NB_HOME \ - && jupyter lab clean -y \ - && conda clean --yes --all --force-pkgs-dirs \ - && rm -rf /home/$NB_USER/.cache/pip \ && rm -rf /tmp/downloaded_packages/ /tmp/*.rds \ && rm -rf $GEM_HOME/cache \ && rm -rf /usr/local/bundle/cache \ @@ -142,8 +154,5 @@ RUN cd $NB_HOME \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* \ && rm -rf $HOME/scripts -RUN fix-permissions $HOME \ - && fix-permissions $CONDA_DIR -# Switch back to jovyan to avoid accidental container runs as root USER $NB_UID diff --git a/env/dev/install_decktape.sh b/env/dev/install_decktape.sh index fa8e0b6..47aa938 100644 --- a/env/dev/install_decktape.sh +++ b/env/dev/install_decktape.sh @@ -63,6 +63,7 @@ apt-get update -qq \ libcups2 \ libxss-dev \ libxrandr-dev \ + && rm -rf /var/lib/apt/lists/* \ && apt-get autoclean \ && apt-get autoremove \ && apt-get clean diff --git a/env/dev/install_jekyll.sh b/env/dev/install_jekyll.sh index 60806e6..9d49326 100644 --- a/env/dev/install_jekyll.sh +++ b/env/dev/install_jekyll.sh @@ -1,7 +1,10 @@ #!/bin/bash apt-get update \ - && apt-get install -y ruby-full build-essential zlib1g-dev + && apt-get install -y \ + ruby-full \ + build-essential \ + zlib1g-dev # https://github.com/sass-contrib/sass-embedded-host-ruby/issues/130#issuecomment-1588245011 gem install sass-embedded gem install sass --force sass-embedded diff --git a/env/dev/install_vim.sh b/env/dev/install_vim.sh index 1fc248d..323a090 100644 --- a/env/dev/install_vim.sh +++ b/env/dev/install_vim.sh @@ -5,8 +5,12 @@ ################# apt-get update \ - && apt-get install -y vim \ + && apt-get install -y --no-install-recommends vim \ && curl -fLo $HOME/.vim/autoload/plug.vim --create-dirs \ https://raw.githubusercontent.com/junegunn/vim-plug/master/plug.vim \ - && vim +PlugInstall +qall + && vim +PlugInstall +qall \ + && rm -rf /var/lib/apt/lists/* \ + && apt-get autoclean \ + && apt-get autoremove \ + && apt-get clean diff --git a/env/r/install_R.sh b/env/r/install_R.sh index b1ddaaa..31a2505 100644 --- a/env/r/install_R.sh +++ b/env/r/install_R.sh @@ -56,6 +56,8 @@ apt-get update \ && ln -s /usr/lib/R/site-library/littler/examples/testInstalled.r /usr/local/bin/testInstalled.r \ && ln -s /usr/lib/R/site-library/littler/examples/update.r /usr/local/bin/update.r \ && rm -rf /tmp/downloaded_packages/ /tmp/*.rds \ -&& rm -rf /var/lib/apt/lists/* - +&& rm -rf /var/lib/apt/lists/* \ +&& apt-get autoclean \ +&& apt-get autoremove \ +&& apt-get clean diff --git a/env/r/install_R_gds.sh b/env/r/install_R_gds.sh index 2190603..417ffba 100644 --- a/env/r/install_R_gds.sh +++ b/env/r/install_R_gds.sh @@ -26,21 +26,6 @@ apt-get update -qq \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -#--- R ---# -# https://github.com/rocker-org/rocker-versioned/blob/master/r-ver/Dockerfile -# Look at: http://sites.psu.edu/theubunturblog/installing-r-in-ubuntu/ - -echo "deb https://cloud.r-project.org/bin/linux/ubuntu focal-cran40/" >> /etc/apt/sources.list \ - && sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys E298A3A825C0D65DFD57CBB651716619E084DAB9 \ - && apt-get update \ - && apt-get install -y \ - r-base \ - r-base-dev \ - && apt-get autoclean \ - && apt-get autoremove \ - && apt-get clean \ - && rm -rf /var/lib/apt/lists/* - #--- R libs ---# # SAN requirements (on top of Rocker Geospatial) # https://github.com/GDSL-UL/san/blob/master/01-overview.Rmd diff --git a/env/r/setup_py-r.sh b/env/r/setup_py-r.sh index 9a54049..c917d92 100644 --- a/env/r/setup_py-r.sh +++ b/env/r/setup_py-r.sh @@ -1,3 +1,5 @@ +# Set up links between Python/R (IRkernel, rpy2) and R/Python (reticulate) + ln -s /opt/conda/envs/gds/bin/jupyter /usr/local/bin R -e "install.packages(c( \ 'IRkernel', 'reticulate' \ From e1fc79003b0a2f45b431cb26fe9abfb7bd0e996f Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Mon, 10 Jun 2024 12:45:58 +0100 Subject: [PATCH 06/11] Move more py logic to scripts from Dockerfile + mv scripts from r to dev as better fit --- env/Dockerfile | 26 +---------------------- env/dev/install_jekyll.sh | 8 ++++++- env/{r => dev}/install_quarto.sh | 0 env/{r => dev}/install_tippecanoe.sh | 0 env/py/install_jupyter_dev.sh | 23 +++++++++++++++++++++ env/py/install_py_stack.sh | 31 ++++++++++++++++++++++++++++ env/r/setup_py-r.sh | 3 ++- 7 files changed, 64 insertions(+), 27 deletions(-) rename env/{r => dev}/install_quarto.sh (100%) rename env/{r => dev}/install_tippecanoe.sh (100%) create mode 100644 env/py/install_jupyter_dev.sh create mode 100644 env/py/install_py_stack.sh diff --git a/env/Dockerfile b/env/Dockerfile index 9742653..f9a43a4 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -19,31 +19,7 @@ USER $NB_UID #--- Python ---# ADD ./py/gds_py.yml /home/${NB_USER}/ -RUN mamba env create -f gds_py.yml \ - && source activate gds \ - && python -m ipykernel install --user --name gds --display-name "GDS-$GDS_ENV_VERSION" \ - && conda deactivate \ - && rm ./gds_py.yml \ - && conda clean --yes --all --force-pkgs-dirs \ - && find /opt/conda/ -follow -type f -name '*.a' -delete \ - && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ - && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ - && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ - -follow -type f -name '*.js' ! -name '*.min.js' -delete \ - && pip cache purge \ - && rm -rf $HOME/.cache/pip - -# Make GDS default -RUN jupyter lab --generate-config \ - && echo "c.MultiKernelManager.default_kernel_name='gds'" >> \ - /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ - && echo "conda activate gds" >> /home/${NB_USER}/.bashrc \ - && echo "c.KernelSpecManager.ensure_native_kernel = False" >> \ - /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ -# https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982 - && echo "c.KernelSpecManager.whitelist = {'gds'}" >> \ - /home/${NB_USER}/.jupyter/jupyter_lab_config.py -RUN jupyter kernelspec remove -y python3 +RUN $HOME/scripts/install_py_stack.sh ENV PATH="/opt/conda/envs/gds/bin/:${PATH}" #--- Jupyter Dev ---# diff --git a/env/dev/install_jekyll.sh b/env/dev/install_jekyll.sh index 9d49326..361daa9 100644 --- a/env/dev/install_jekyll.sh +++ b/env/dev/install_jekyll.sh @@ -1,7 +1,7 @@ #!/bin/bash apt-get update \ - && apt-get install -y \ + && apt-get install -y --no-install-recommends \ ruby-full \ build-essential \ zlib1g-dev @@ -10,3 +10,9 @@ gem install sass-embedded gem install sass --force sass-embedded gem install jekyll bundler github-pages jekyll-scholar just-the-docs +rm -rf /var/lib/gems/3.0.0/cache/* +rm -rf /var/lib/apt/lists/* \ + && apt-get autoclean \ + && apt-get autoremove \ + && apt-get clean + diff --git a/env/r/install_quarto.sh b/env/dev/install_quarto.sh similarity index 100% rename from env/r/install_quarto.sh rename to env/dev/install_quarto.sh diff --git a/env/r/install_tippecanoe.sh b/env/dev/install_tippecanoe.sh similarity index 100% rename from env/r/install_tippecanoe.sh rename to env/dev/install_tippecanoe.sh diff --git a/env/py/install_jupyter_dev.sh b/env/py/install_jupyter_dev.sh new file mode 100644 index 0000000..59149ea --- /dev/null +++ b/env/py/install_jupyter_dev.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +#--- [Run under $NB_USER] ---# + +#--- Text editor / default Vim ---# +# https://github.com/jupyterlab/jupyterlab/issues/14599 +mkdir -p /home/$NB_USER/.jupyter/lab/user-settings/\@jupyterlab/fileeditor-extension/ \ + && echo '{"editorConfig": {"codeFolding": true, "highlightActiveLine": true, "highlightTrailingWhitespace": true}}' \ + >> /home/$NB_USER/.jupyter/lab/user-settings/\@jupyterlab/fileeditor-extension/plugin.jupyterlab-settings + +#--- Dask Jupyter extension & Bash kernel ---# +pip install dask-labextension bash_kernel jupyterlab_vim \ + && python -m bash_kernel.install \ + && mamba install python-graphviz \ + && pip cache purge \ + && conda clean --all --yes --force-pkgs-dirs + +sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'ir'}/c.KernelSpecManager.whitelist = {'gds', 'ir', 'bash'}/g" \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py + +#--- Pypeteer ---# +pyppeteer-install + diff --git a/env/py/install_py_stack.sh b/env/py/install_py_stack.sh new file mode 100644 index 0000000..33cc8df --- /dev/null +++ b/env/py/install_py_stack.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +#--- [Run under $NB_USER] ---# + +# Install GDS conda environment +mamba env create -f gds_py.yml \ + && source activate gds \ + && python -m ipykernel install --user --name gds --display-name "GDS-$GDS_ENV_VERSION" \ + && conda deactivate \ + && rm ./gds_py.yml \ + && conda clean --yes --all --force-pkgs-dirs \ + && find /opt/conda/ -follow -type f -name '*.a' -delete \ + && find /opt/conda/ -follow -type f -name '*.pyc' -delete \ + && find /opt/conda/ -follow -type f -name '*.js.map' -delete \ + && find /opt/conda/envs/gds//lib/python*/site-packages/bokeh/server/static \ + -follow -type f -name '*.js' ! -name '*.min.js' -delete \ + && pip cache purge \ + && rm -rf $HOME/.cache/pip + +# Make GDS default +RUN jupyter lab --generate-config \ + && echo "c.MultiKernelManager.default_kernel_name='gds'" >> \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ + && echo "conda activate gds" >> /home/${NB_USER}/.bashrc \ + && echo "c.KernelSpecManager.ensure_native_kernel = False" >> \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ +# https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982 + && echo "c.KernelSpecManager.whitelist = {'gds'}" >> \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py +RUN jupyter kernelspec remove -y python3 + diff --git a/env/r/setup_py-r.sh b/env/r/setup_py-r.sh index c917d92..8d9a8ea 100644 --- a/env/r/setup_py-r.sh +++ b/env/r/setup_py-r.sh @@ -20,7 +20,8 @@ pip install -U --no-deps \ pytz_deprecation_shim \ jinja2 \ 'cffi>=1.10.0' \ - tzlocal + tzlocal \ + && pip cache purge rm -rf /home/$NB_USER/.cache/pip rm -rf /tmp/downloaded_packages/ /tmp/*.rds From 9ae52bc193c64ae1010963bc1a35c6d9e1886240 Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Mon, 10 Jun 2024 12:56:33 +0100 Subject: [PATCH 07/11] minor fix on py move to scripts --- env/py/install_py_stack.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/env/py/install_py_stack.sh b/env/py/install_py_stack.sh index 33cc8df..ede34fd 100644 --- a/env/py/install_py_stack.sh +++ b/env/py/install_py_stack.sh @@ -18,14 +18,14 @@ mamba env create -f gds_py.yml \ && rm -rf $HOME/.cache/pip # Make GDS default -RUN jupyter lab --generate-config \ +# https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982 +jupyter lab --generate-config \ && echo "c.MultiKernelManager.default_kernel_name='gds'" >> \ /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ && echo "conda activate gds" >> /home/${NB_USER}/.bashrc \ && echo "c.KernelSpecManager.ensure_native_kernel = False" >> \ /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ -# https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982 && echo "c.KernelSpecManager.whitelist = {'gds'}" >> \ /home/${NB_USER}/.jupyter/jupyter_lab_config.py -RUN jupyter kernelspec remove -y python3 +jupyter kernelspec remove -y python3 From 4f8eb0e8c7554aa02ad8e9fd89e02a04d89fbf70 Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Thu, 27 Jun 2024 13:48:20 +0000 Subject: [PATCH 08/11] Add compose demo for Code --- frontend_code/Dockerfile | 2 +- frontend_code/compose.yml | 16 ++++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) create mode 100644 frontend_code/compose.yml diff --git a/frontend_code/Dockerfile b/frontend_code/Dockerfile index 0a67896..4ba9edd 100644 --- a/frontend_code/Dockerfile +++ b/frontend_code/Dockerfile @@ -1,4 +1,4 @@ -FROM darribas/gds_dev:10.0 +FROM gds:11.0alpha USER root diff --git a/frontend_code/compose.yml b/frontend_code/compose.yml new file mode 100644 index 0000000..68a5a80 --- /dev/null +++ b/frontend_code/compose.yml @@ -0,0 +1,16 @@ +--- +services: + gds_code: + image: gds_code:11.0alpha + container_name: gds_code + environment: + - TZ=Etc/UTC + - PASSWORD= #optional + - DEFAULT_WORKSPACE=/home/jovyan #optional + volumes: + - /HOST_PATH/:/home/jovyan/work + ports: + - 8443:8080 + - 4000:4000 + restart: unless-stopped + From 721b96bb46f801184b9f798558aece149ae661d3 Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Sat, 29 Jun 2024 07:28:08 +0000 Subject: [PATCH 09/11] Fixing issues on python build --- env/Dockerfile | 13 ++++++------- env/py/gds_py.yml | 3 --- env/py/install_jupyter_dev.sh | 30 +++++++++++++++++++----------- env/py/install_py_stack.sh | 5 ++++- env/r/setup_py-r.sh | 2 +- 5 files changed, 30 insertions(+), 23 deletions(-) diff --git a/env/Dockerfile b/env/Dockerfile index f9a43a4..b8896a5 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -16,23 +16,22 @@ USER $NB_UID ############## ### Python ### ############## + +#--- Jupyter Dev ---# +RUN $HOME/scripts/install_jupyter_dev.sh + #--- Python ---# ADD ./py/gds_py.yml /home/${NB_USER}/ RUN $HOME/scripts/install_py_stack.sh ENV PATH="/opt/conda/envs/gds/bin/:${PATH}" -#--- Jupyter Dev ---# -RUN $HOME/scripts/install_jupyter_dev.sh - -#--- Jupyter config ---# +#--- Clean up ---# USER root -# Turn off notifications -RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \ # Clean cache up - && jupyter lab clean -y \ +RUN jupyter lab clean -y \ && conda clean --yes --all --force-pkgs-dirs \ && npm cache clean --force \ && rm -rf $CONDA_DIR/share/jupyter/lab/staging \ diff --git a/env/py/gds_py.yml b/env/py/gds_py.yml index f5b4b94..5f1495f 100644 --- a/env/py/gds_py.yml +++ b/env/py/gds_py.yml @@ -29,7 +29,6 @@ dependencies: - ipympl - ipywidgets - jupyterlab>=4.0 - - jupyter_bokeh - jupyter-book - jupytext - legendgram @@ -46,8 +45,6 @@ dependencies: - pandana - pip - pip: - - jupyterlab-geojson - - jupyterlab_myst - matplotlib-scalebar - polars # - pygeoda diff --git a/env/py/install_jupyter_dev.sh b/env/py/install_jupyter_dev.sh index 59149ea..2227b7c 100644 --- a/env/py/install_jupyter_dev.sh +++ b/env/py/install_jupyter_dev.sh @@ -8,16 +8,24 @@ mkdir -p /home/$NB_USER/.jupyter/lab/user-settings/\@jupyterlab/fileeditor-exten && echo '{"editorConfig": {"codeFolding": true, "highlightActiveLine": true, "highlightTrailingWhitespace": true}}' \ >> /home/$NB_USER/.jupyter/lab/user-settings/\@jupyterlab/fileeditor-extension/plugin.jupyterlab-settings -#--- Dask Jupyter extension & Bash kernel ---# -pip install dask-labextension bash_kernel jupyterlab_vim \ - && python -m bash_kernel.install \ - && mamba install python-graphviz \ - && pip cache purge \ - && conda clean --all --yes --force-pkgs-dirs +#--- JupyterLab extensions & Bash kernel ---# -sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'ir'}/c.KernelSpecManager.whitelist = {'gds', 'ir', 'bash'}/g" \ - /home/${NB_USER}/.jupyter/jupyter_lab_config.py - -#--- Pypeteer ---# -pyppeteer-install +jupyter labextension disable "@jupyterlab/apputils-extension:announcements" \ + && pip install \ + bash_kernel \ + jupyterlab-geojson \ + jupyterlab_myst \ + jupyterlab-quarto \ + jupyterlab_vim \ + jupyterlab_widgets \ + jupytext +# Bash kernel +python -m bash_kernel.install +# Required for dask extension +mamba install python-graphviz +# Clean +pip cache purge \ + && conda clean --all --yes --force-pkgs-dirs \ + && jupyter lab clean -y \ + && npm cache clean --force diff --git a/env/py/install_py_stack.sh b/env/py/install_py_stack.sh index ede34fd..f005069 100644 --- a/env/py/install_py_stack.sh +++ b/env/py/install_py_stack.sh @@ -17,6 +17,9 @@ mamba env create -f gds_py.yml \ && pip cache purge \ && rm -rf $HOME/.cache/pip +# Pypeteer +pyppeteer-install + # Make GDS default # https://github.com/jupyter/notebook/issues/3674#issuecomment-397212982 jupyter lab --generate-config \ @@ -25,7 +28,7 @@ jupyter lab --generate-config \ && echo "conda activate gds" >> /home/${NB_USER}/.bashrc \ && echo "c.KernelSpecManager.ensure_native_kernel = False" >> \ /home/${NB_USER}/.jupyter/jupyter_lab_config.py \ - && echo "c.KernelSpecManager.whitelist = {'gds'}" >> \ + && echo "c.KernelSpecManager.whitelist = {'gds', 'bash'}" >> \ /home/${NB_USER}/.jupyter/jupyter_lab_config.py jupyter kernelspec remove -y python3 diff --git a/env/r/setup_py-r.sh b/env/r/setup_py-r.sh index 8d9a8ea..024549a 100644 --- a/env/r/setup_py-r.sh +++ b/env/r/setup_py-r.sh @@ -10,7 +10,7 @@ LD_LIBRARY_PATH /usr/local/lib/R/lib/:${LD_LIBRARY_PATH} fix-permissions $HOME fix-permissions $CONDA_DIR -sed -i "s/c.KernelSpecManager.whitelist = {'gds'}/c.KernelSpecManager.whitelist = {'gds', 'ir'}/g" \ +sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'bash'}/c.KernelSpecManager.whitelist = {'gds', 'bash', 'ir'}/g" \ me/${NB_USER}/.jupyter/jupyter_lab_config.py pip install -U --no-deps \ From c09a1445a4507fea187402fb85391adb3ccda8ea Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Sat, 29 Jun 2024 20:01:41 +0000 Subject: [PATCH 10/11] Fix R kernel add + remove geo libs to pull from conda --- env/Dockerfile | 8 ++++---- env/r/install_R_stack.sh | 6 +++--- env/r/setup_py-r.sh | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/env/Dockerfile b/env/Dockerfile index b8896a5..180d8fc 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -53,9 +53,9 @@ RUN source activate gds \ USER root # Remove Conda from path to not interfere with R install -RUN echo ${PATH} -ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -RUN echo ${PATH} +#RUN echo ${PATH} +#ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +#RUN echo ${PATH} # install R RUN $HOME/scripts/install_R.sh @@ -67,7 +67,7 @@ RUN $HOME/scripts/install_R_stack.sh RUN $HOME/scripts/install_R_gds.sh # re-attach conda to path -ENV PATH="/opt/conda/envs/gds/bin/:/opt/conda/condabin:/opt/conda/envs/gds/bin:/opt/conda/bin:${PATH}" +#ENV PATH="/opt/conda/envs/gds/bin/:/opt/conda/condabin:/opt/conda/envs/gds/bin:/opt/conda/bin:${PATH}" #--- R/Python ---# diff --git a/env/r/install_R_stack.sh b/env/r/install_R_stack.sh index 2f50129..e07a4b0 100644 --- a/env/r/install_R_stack.sh +++ b/env/r/install_R_stack.sh @@ -60,8 +60,8 @@ add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable \ gdal-bin \ lbzip2 \ libfftw3-dev \ - libgdal-dev \ - libgeos-dev \ + #libgdal-dev \ + #libgeos-dev \ libgsl0-dev \ libgl1-mesa-dev \ libglu1-mesa-dev \ @@ -69,7 +69,7 @@ add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable \ libhdf5-dev \ libjq-dev \ libpq-dev \ - libproj-dev \ + #libproj-dev \ libprotobuf-dev \ libnetcdf-dev \ libsqlite3-dev \ diff --git a/env/r/setup_py-r.sh b/env/r/setup_py-r.sh index 024549a..2216699 100644 --- a/env/r/setup_py-r.sh +++ b/env/r/setup_py-r.sh @@ -10,8 +10,8 @@ LD_LIBRARY_PATH /usr/local/lib/R/lib/:${LD_LIBRARY_PATH} fix-permissions $HOME fix-permissions $CONDA_DIR -sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'bash'}/c.KernelSpecManager.whitelist = {'gds', 'bash', 'ir'}/g" \ - me/${NB_USER}/.jupyter/jupyter_lab_config.py +sed -i "s/c.KernelSpecManager.whitelist = {'gds', 'bash'}/c.KernelSpecManager.whitelist = {'gds', 'ir', 'bash'}/g" \ + /home/${NB_USER}/.jupyter/jupyter_lab_config.py pip install -U --no-deps \ rpy2 \ From a3e33f35b5444b12fef3de4be551fe4689c3654c Mon Sep 17 00:00:00 2001 From: Dani Arribas-Bel Date: Tue, 2 Jul 2024 12:22:40 +0000 Subject: [PATCH 11/11] Fine tune mono-structure. Not working but ready to merge --- env/Dockerfile | 8 ++++---- env/r/install_R.sh | 1 + env/r/install_R_stack.sh | 8 +++++--- 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/env/Dockerfile b/env/Dockerfile index 180d8fc..b8896a5 100644 --- a/env/Dockerfile +++ b/env/Dockerfile @@ -53,9 +53,9 @@ RUN source activate gds \ USER root # Remove Conda from path to not interfere with R install -#RUN echo ${PATH} -#ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" -#RUN echo ${PATH} +RUN echo ${PATH} +ENV PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +RUN echo ${PATH} # install R RUN $HOME/scripts/install_R.sh @@ -67,7 +67,7 @@ RUN $HOME/scripts/install_R_stack.sh RUN $HOME/scripts/install_R_gds.sh # re-attach conda to path -#ENV PATH="/opt/conda/envs/gds/bin/:/opt/conda/condabin:/opt/conda/envs/gds/bin:/opt/conda/bin:${PATH}" +ENV PATH="/opt/conda/envs/gds/bin/:/opt/conda/condabin:/opt/conda/envs/gds/bin:/opt/conda/bin:${PATH}" #--- R/Python ---# diff --git a/env/r/install_R.sh b/env/r/install_R.sh index 31a2505..c3497fd 100644 --- a/env/r/install_R.sh +++ b/env/r/install_R.sh @@ -14,6 +14,7 @@ apt-get update -qq \ && apt-get update -qq \ && apt-get install -y --no-install-recommends \ ca-certificates \ + libcurl4-openssl-dev \ less \ libopenblas-base \ locales \ diff --git a/env/r/install_R_stack.sh b/env/r/install_R_stack.sh index e07a4b0..396639d 100644 --- a/env/r/install_R_stack.sh +++ b/env/r/install_R_stack.sh @@ -32,6 +32,7 @@ apt-get update -qq \ R -e "install.packages(c( \ 'tidyverse', \ 'devtools', \ + 'curl', \ 'rmarkdown', \ 'BiocManager', \ 'vroom', \ @@ -60,16 +61,17 @@ add-apt-repository -y ppa:ubuntugis/ubuntugis-unstable \ gdal-bin \ lbzip2 \ libfftw3-dev \ - #libgdal-dev \ - #libgeos-dev \ + libgdal-dev \ + libgeos-dev \ libgsl0-dev \ libgl1-mesa-dev \ libglu1-mesa-dev \ libhdf4-alt-dev \ libhdf5-dev \ libjq-dev \ + liblwgeom-dev \ libpq-dev \ - #libproj-dev \ + libproj-dev \ libprotobuf-dev \ libnetcdf-dev \ libsqlite3-dev \