before moving 2 g2

vrodriguezf · Oct 15, 2024 · 4364ac9 · 4364ac9
1 parent 0775403
commit 4364ac9
Show file tree

Hide file tree

Showing 44 changed files with 6,359 additions and 2,150 deletions.
diff --git a/docker/Dockerfile.jupyter b/docker/Dockerfile.jupyter
@@ -8,7 +8,7 @@
 # Base image #
 ##############
 ##--- Setup Ubuntu
-FROM misantamaria/dvats-jupyter:cuda12.2.0-ubuntu20.04-hooks-mplots
+FROM misantamaria/dvats-jupyter:cuda12.2.0-ubuntu20.04-hooks-mplots-zeroshot
 
 #--- Tags
 LABEL maintainer="vrodriguezf <victor.rfernandez@upm.es>"
@@ -95,7 +95,9 @@ ARG JUPYTER_TOKEN
 ENV JUPYTER_TOKEN=${JUPYTER_TOKEN}
 # default command will be to launch JupyterLab server for development
 
-COPY --chown=$UID:$GID  ./ $HOME/work
+#COPY --chown=$UID:$GID  ./ $HOME/work
+RUN mkdir -p $HOME/work
+COPY --chown=$UID:$GID ./.git $HOME/work/.git
 
 RUN  source /usr/local/share/miniconda3/bin/activate env \
      && cd $HOME/work \
@@ -116,6 +118,52 @@ RUN conda list octave
 
 RUN pip install aeon
 
+# #############
+# # Zero-shot #
+# #############
+# # POETRY
+# RUN curl -sSL https://install.python-poetry.org | ${ENV_PREFIX}/bin/python3.10 -
+# ENV POETRY_PATH="${HOME}/.local/bin/poetry"
+# RUN ${POETRY_PATH} --version
+
+# # MOMENT
+# RUN /usr/local/share/miniconda3/envs/env/bin/pip install git+https://github.com/moment-timeseries-foundation-model/moment.git
+
+# # MOIRAI
+# ENV LIB_PATH /usr/local/share/lib
+# RUN mkdir $LIB_PATH
+# RUN conda activate $ENV_PREFIX \
+#     && cd $LIB_PATH \
+#     && rm -rf uni2ts \
+#     && git clone https://github.com/SalesforceAIResearch/uni2ts.git \
+#     && cd uni2ts \
+#     && pip  install -e '.[notebook]' --no-warn-script-location \
+#     && conda deactivate
+
+# RUN conda activate $ENV_PREFIX \
+#     && cd $LIB_PATH \
+#     && rm -rf TEMPO TEMPO_project \
+#     && git clone https://github.com/DC-research/TEMPO.git \
+#     && cd TEMPO \
+#     && mv requirements.txt _requirements.txt \
+#     && grep -v -- '--extra-index-url' _requirements.txt > requirements.txt \
+#     && mv requirements.txt __requirements.txt \
+#     && grep -v -- '#' __requirements.txt > requirements.txt \
+#     && ${POETRY_PATH} init --name TEMPO --description "TEMPO" --author "Author Name <email@example.com>" --license "MIT" --no-interaction  \
+#     && cd TEMPO \
+#     && cat requirements.txt | xargs -I {} ${POETRY_PATH} add {} --no-interaction \
+#     # Lock versions
+#     && ${POETRY_PATH} lock --no-interaction \
+#     # Install dependencies
+#     && ${POETRY_PATH} install --no-interaction \
+#     # Ensure modules 
+#     && find ./TEMPO -type d -exec touch {}/__init__.py \; \
+#     && ${POETRY_PATH} new --name TEMPO --src TEMPO_project  \
+#     && cd TEMPO_project \
+#     && /usr/local/share/miniconda3/envs/env/bin/pip  install -e . \
+#     && conda deactivate
+# # ---- Zero shot ---- #
+
 ARG JUPYTER_TOKEN
 ENV JUPYTER_TOKEN=${JUPYTER_TOKEN}
 

diff --git a/docker/Dockerfile.jupyter.zeroshot b/docker/Dockerfile.jupyter.zeroshot
@@ -0,0 +1,85 @@
+#####################################################################
+# CONDA & MINICONDA WITH PYBASE DOCKERFILE                          #
+# For zeroshot models                                               #
+# dvats-jupyter:cuda12.2.0-ubuntu20.04-hooks-mplots-zeroshot        #
+#####################################################################
+##############
+# Base image #
+##############
+##--- Setup Ubuntu
+FROM misantamaria/dvats-jupyter:cuda12.2.0-ubuntu20.04-hooks-mplots
+
+##--- Setup bash
+SHELL [ "/bin/bash", "--login", "-c" ]
+
+COPY docker/environment.zeroshot.yml /tmp/environment.zeroshot.yml
+
+RUN mamba env update --name env --file /tmp/environment.zeroshot.yml \
+    && mamba clean --all --yes
+
+#############
+# Zero-shot #
+#############
+# POETRY
+RUN curl -sSL https://install.python-poetry.org | ${ENV_PREFIX}/bin/python3.10 -
+ENV POETRY_PATH="${HOME}/.local/bin/poetry"
+RUN ${POETRY_PATH} --version
+
+# MOMENT
+ENV PIP $CONDA_DIR/envs/env/bin/pip
+RUN $PIP install git+https://github.com/moment-timeseries-foundation-model/moment.git         
+#RUN $PIP install git+https://github.com/moment-timeseries-foundation-model/moment.git@040df9a5091eb83201d8764acb96350a5c84f47e
+# Commit usado: 040df9a5091eb83201d8764acb96350a5c84f47e
+#RUN $PIP install git+https://github.com/misantamaria/moment.git@v1.0.0
+ENV LIB_PATH /usr/local/share/lib
+ENV PKGS $ENV_PREFIX/lib/python3.10/site-packages/
+RUN mkdir $LIB_PATH
+#RUN conda init bash \
+#    && /bin/bash -c "source $CONDA_DIR/etc/profile.d/conda.sh" \
+#    && cd $LIB_PATH\
+#    && rm -rf moment \
+#    && git clone https://github.com/misantamaria/moment.git \
+#    && cd moment \
+#    && git checkout v1.0.0 \
+#    && $PIP  install -e '.[notebook]' --no-warn-script-location --no-deps --target=$PKGS
+
+# MOIRAI
+RUN conda init bash \
+    && /bin/bash -c "source $CONDA_DIR/etc/profile.d/conda.sh" \
+    && cd $LIB_PATH \
+    && rm -rf uni2ts \
+    && git clone https://github.com/misantamaria/uni2ts.git \
+    && cd uni2ts \
+    && git checkout 1.1.2 \
+    && $PIP  install -e '.[notebook]' --no-warn-script-location --no-deps --target=$PKGS
+
+#TEMPO
+RUN conda init bash \
+    && /bin/bash -c "source $CONDA_DIR/etc/profile.d/conda.sh" \
+    && cd $LIB_PATH \
+    && rm -rf TEMPO TEMPO_project \
+    && git clone https://github.com/DC-research/TEMPO.git \
+    #&& git clone https://github.com/misantamaria/TEMPO.git \
+    && cd TEMPO \
+    #&& git checkout v1.0.0 \
+    && git checkout 755bcc765a0b6e9921063f5abded34dd900b2f14 \
+    && mv requirements.txt _requirements.txt \
+    && grep -v -- '--extra-index-url' _requirements.txt > requirements.txt \
+    && mv requirements.txt __requirements.txt \
+    && grep -v -- '#' __requirements.txt > requirements.txt \
+    && ${POETRY_PATH} init --name TEMPO --description "TEMPO" --author "Author Name <email@example.com>" --license "MIT" --no-interaction  \
+    && cat requirements.txt | xargs -I {} ${POETRY_PATH} add {} --no-interaction \
+    # Lock versions
+    && ${POETRY_PATH} lock --no-interaction \
+    # Install dependencies
+    && ${POETRY_PATH} install --no-interaction \
+    # Ensure modules 
+    && cd .. \
+    && find ./TEMPO -type d -exec touch {}/__init__.py \; \
+    && ${POETRY_PATH} new --name TEMPO --src TEMPO_project  \
+    && cd TEMPO_project \
+    && $PIP install -e . --no-deps  --target=$PKGS 
+
+RUN mkdir -p ${LIB_PATH}/TEMPO/TEMPO_checkpoints \
+    && chmod -R 777 /usr/local/share/lib/TEMPO/
+    # ---- Zero shot ---- #
diff --git a/docker/Dockerfile.jupyter_for_when_zero_shot_corrected b/docker/Dockerfile.jupyter_for_when_zero_shot_corrected
@@ -0,0 +1,130 @@
+#############################
+# JUPYTER VOLUME DOCKERFILE #
+#***************************#
+# Set up a jupyter-lab for  #
+# developing using deepvats #
+#############################
+##############
+# Base image #
+##############
+##--- Setup Ubuntu
+FROM misantamaria/dvats-jupyter:cuda12.2.0-ubuntu20.04-hooks-mplots-zeroshot
+
+#--- Tags
+LABEL maintainer="vrodriguezf <victor.rfernandez@upm.es>"
+LABEL cuda_version=${CUDA_VERSION}
+
+##---Initialize bash
+SHELL [ "/bin/bash", "--login", "-c" ]
+
+#-- Environmental variables for wandb
+ENV LC_ALL=C.UTF-8 \
+    LANG=C.UTF-8 
+
+
+#-- Octave GUI - uing interactive octave plots
+RUN apt-get update \
+    && DEBIAN_FRONTEND=noninteractive  apt-get install -y \
+    xvfb \
+    x11-xserver-utils \
+    gnuplot \
+    && rm -rf /var/lib/apt/lists/*
+
+
+########################
+# Create non-root user #
+########################
+ARG username=user\
+    uid=1000 \
+    gid=1000
+ENV USER=$username \
+    UID=$uid \
+    GID=$gid 
+ENV HOME=/home/$USER
+
+#-- Passwordless sudo (TODO: mv to root section)
+RUN addgroup --gid $GID $USER  \
+    && adduser --disabled-password \
+    --gecos "Non-root user" \
+    --uid $UID \
+    --gid $GID \
+    --home $HOME $USER\
+    && echo "$USER ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
+
+#RUN sudo chown -R $UID:$GID /usr/local/share/miniconda3/envs/env
+
+#-- Move to user
+USER $USER
+
+# Add the jupyterlab settings
+COPY --chown=$uid:$gid docker/jupyter_config $HOME/.jupyter
+
+#############################
+# Ensure miniconda for user #
+#############################
+
+#ENV MINICONDA_VERSION=4.10.3 \ 
+#    CONDA_DIR=/opt/miniconda3 
+
+#ENV PATH=$CONDA_DIR/bin:$PATH \
+ENV PROJECT_DIR=${HOME}
+#ENV ENV_PREFIX=/opt/env
+
+RUN echo "--> Activate conda"
+
+# Make conda activate command available from /bin/bash --interative shells
+RUN  conda init bash  \
+    # Create a project directory inside user home
+    && mkdir -p $PROJECT_DIR 
+
+RUN echo "--> WORKDIR"
+WORKDIR ${HOME}
+
+COPY --chown=$UID:$GID docker/entrypoint.sh /usr/local/bin
+
+# Make bash automatically activate the conda environment
+RUN echo "conda activate $ENV_PREFIX" >> ~/.bashrc \
+    && echo "--> Exec entrypoint" \
+    && chmod u+x /usr/local/bin/entrypoint.sh
+
+ENV PATH /usr/local/share/miniconda3/envs/env/bin:$PATH
+
+ENTRYPOINT [ "/usr/local/bin/entrypoint.sh" ]
+
+ARG JUPYTER_TOKEN
+ENV JUPYTER_TOKEN=${JUPYTER_TOKEN}
+# default command will be to launch JupyterLab server for development
+
+COPY --chown=$UID:$GID  ./ $HOME/work
+
+RUN  source /usr/local/share/miniconda3/bin/activate env \
+     && cd $HOME/work \
+     && ls -la $HOME/work \
+     && git --version 
+
+SHELL [ "/bin/sh", "-c" ]
+RUN cd ${HOME}/work/ \
+    && git config --global --add safe.directory /home/macu/work \
+    && git rev-parse --is-inside-work-tree || (echo "Not inside a Git repository"; exit 1) \
+    && pre-commit install
+
+COPY docker/pre-commit .git/hooks/pre-commit
+
+RUN conda list octave
+
+#CMD ["sh", "-c", "jupyter lab --no-browser --ip 0.0.0.0 --ContentsManager.allow_hidden=True --NotebookApp.token=$JUPYTER_TOKEN"]
+
+RUN pip install aeon
+
+ARG JUPYTER_TOKEN
+ENV JUPYTER_TOKEN=${JUPYTER_TOKEN}
+
+CMD [\
+    "sh", "-c", \
+    "xvfb-run -a jupyter lab \
+    --no-browser \
+    --ip 0.0.0.0 \
+    --ContentsManager.allow_hidden=True \
+    --NotebookApp.token=$JUPYTER_TOKEN"\
+    ]
+
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
@@ -27,6 +27,7 @@ services:
       - ${LOCAL_DATA_PATH}:/home/${USER_NAME}/data/
       - conda-env:/usr/local/share/miniconda3/envs/env
       - miniconda:/usr/local/share/miniconda3
+      - lib:/usr/local/share/lib
     init: true
     stdin_open: true
     tty: true
@@ -44,8 +45,10 @@ services:
           dockerfile: docker/Dockerfile.rstudio
           args:
               - WANDB_API_KEY=${WANDB_API_KEY} #
-              - RETICULATE_PYTHON_ENV=/home/env
-              - RETICULATE_MINICONDA_PATH=/home/miniconda3
+              #- RETICULATE_PYTHON_ENV=/home/env
+              #- RETICULATE_MINICONDA_PATH=/home/miniconda3
+              - RETICULATE_PYTHON_ENV=/usr/local/share/miniconda3/envs/env
+              - RETICULATE_MINICONDA_PATH=/usr/local/share/miniconda3
               - USER=${USER_NAME} #* 
               - UID=${USER_ID} #*
               - GID=${GROUP_ID} #*
@@ -72,10 +75,8 @@ services:
          - ${LOCAL_DATA_PATH}:/home/${USER_NAME}/data/ 
          - ../dvats:/home/${USER_NAME}/dvats
          - conda-env:/usr/local/share/miniconda3/envs/env
-         - miniconda:/home/user/local/share/miniconda3 
-         # arreglar bien el error: /home/macu/work/nbs/mplot: No such file or directory
-         #- ../nbs/mplot/:/home/${USER_NAME}/work/nbs/mplot
-         ####
+         - miniconda:/usr/local/share/miniconda3 
+         - lib:/usr/local/share/lib
       deploy:
         resources:
           #limits:

diff --git a/docker/environment.zeroshot.yml b/docker/environment.zeroshot.yml
@@ -0,0 +1,22 @@
+name: env
+channels: 
+- conda-forge
+- anaconda 
+- huggingface
+dependencies:
+  #-- Mplots
+  - gluonts=0.14.4
+  - tqdm=4.66.2
+  - huggingface_hub=0.24.0
+  - transformers=4.33.3
+  - einops=0.7.*
+  - lightning>=2.0
+  - jaxtyping>=0.2.24
+  - python-dotenv=1.0.0
+  - hydra-core=1.3
+  - orjson
+  - tensorboard
+  - multiprocess
+  - safetensors
+  - datasets~=2.17.1
+  - jax[cpu]
diff --git a/dvats/_modidx.py b/dvats/_modidx.py
@@ -72,7 +72,10 @@
                                'dvats.encoder.get_enc_embs_moment_reconstruction': ( 'encoder.html#get_enc_embs_moment_reconstruction',
                                                                                      'dvats/encoder.py'),
                                'dvats.encoder.get_enc_embs_set_stride_set_batch_size': ( 'encoder.html#get_enc_embs_set_stride_set_batch_size',
-                                                                                         'dvats/encoder.py')},
+                                                                                         'dvats/encoder.py'),
+                               'dvats.encoder.kwargs_to_cpu_': ('encoder.html#kwargs_to_cpu_', 'dvats/encoder.py'),
+                               'dvats.encoder.kwargs_to_gpu_': ('encoder.html#kwargs_to_gpu_', 'dvats/encoder.py'),
+                               'dvats.encoder.watch_gpu': ('encoder.html#watch_gpu', 'dvats/encoder.py')},
             'dvats.imports': {},
             'dvats.load': { 'dvats.load.TSArtifact': ('load.html#tsartifact', 'dvats/load.py'),
                             'dvats.load.TSArtifact.__init__': ('load.html#__init__', 'dvats/load.py'),
@@ -226,6 +229,7 @@
                              'dvats.utils.get_wandb_artifacts': ('utils.html#get_wandb_artifacts', 'dvats/utils.py'),
                              'dvats.utils.normalize_columns': ('utils.html#normalize_columns', 'dvats/utils.py'),
                              'dvats.utils.plot_with_dots': ('utils.html#plot_with_dots', 'dvats/utils.py'),
+                             'dvats.utils.print_flush': ('utils.html#print_flush', 'dvats/utils.py'),
                              'dvats.utils.py_function': ('utils.html#py_function', 'dvats/utils.py'),
                              'dvats.utils.remove_constant_columns': ('utils.html#remove_constant_columns', 'dvats/utils.py'),
                              'dvats.utils.show_sequence': ('utils.html#show_sequence', 'dvats/utils.py'),