diff --git a/.copier-config.yaml b/.copier-config.yaml new file mode 100644 index 0000000..dfe8eb4 --- /dev/null +++ b/.copier-config.yaml @@ -0,0 +1,58 @@ +# Changes here will be overwritten by Copier; do NOT edit manually +_commit: v0.4.28 +_src_path: gh:entelecheia/hyperfast-python-template +author: Young Joon Lee +build_and_release: false +code_template_answers_file: .copier-hyfi-config.yaml +code_template_source: null +copyright_year: 2023 +documentaion_tool: mkdocs +documentation_url: https://hyperfast-docker.entelecheia.ai +email: entelecheia@hotmail.com +favicon_path: https://assets.entelecheia.ai/favicon.png +friendly_name: Hyperfast Docker Template +github_repo_name: hyperfast-docker-template +github_username: entelecheia +google_analytics_id: G-0K77490FJT +logo_path: https://assets.entelecheia.ai/logo.png +package_name: hfdk +project_description: 'Hyperfast Docker Template is an innovative project creation + tool that merges the flexibility of Docker with the simplicity of Copier to optimize + the project creation process. This tool allows developers to quickly and efficiently + set up Docker projects, thereby significantly reducing the time and effort traditionally + associated with this process. + + + By harnessing the power of Copier, Hyperfast Docker Template provides a user-friendly + interface that developers can utilize to duplicate Docker projects with minimal + hassle. It eradicates the need for extensive coding knowledge, making Docker more + accessible to a wider range of users. + + + Furthermore, this tool employs a hyperfast approach to Docker project templating, + ensuring that new projects are up and running in the quickest time possible. This + speed does not compromise the quality or functionality of the Docker projects + created, but instead enhances their performance, reliability, and scalability. + + + Hyperfast Docker Template also ensures consistency across Docker projects, as + the use of a template guarantees that all projects conform to a standardized layout + and configuration. This, in turn, enhances the overall workflow, as developers + can familiarize themselves with a consistent project structure and configuration, + leading to increased productivity and efficiency. + + + Whether you''re an experienced developer or a novice just starting out, Hyperfast + Docker Template is the ideal tool to fast-track your Docker project creation process, + ensuring a seamless, efficient, and highly productive development experience.' +project_license: MIT +project_name: hyperfast-docker-template +project_short_description: Hyperfast Docker Template is a powerful tool that leverages + copier to streamline the creation of new Docker projects. It simplifies and accelerates + Docker configurations, fostering a highly efficient and user-friendly development + experience. +upload_to_release: false +upload_to_repository: false +use_launch_buttons: false +use_source_code_skeleton: false + diff --git a/.copier-template/Makefile b/.copier-template/Makefile new file mode 100644 index 0000000..cd2d22b --- /dev/null +++ b/.copier-template/Makefile @@ -0,0 +1,112 @@ +# To do stuff with make, you type `make` in a directory that has a file called +# "Makefile". You can also type `make -f ` to use a different filename. +# +# A Makefile is a collection of rules. Each rule is a recipe to do a specific +# thing, sort of like a grunt task or an npm package.json script. +# +# A rule looks like this: +# +# : +# +# +# The "target" is required. The prerequisites are optional, and the commands +# are also optional, but you have to have one or the other. +# +# Type `make` to show the available targets and a description of each. +# +.DEFAULT_GOAL := help +.PHONY: help +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + + +##@ Clean-up + +clean: ## run all clean commands + @poe clean + +##@ Git Branches + +show-branches: ## show all branches + @git show-branch --list + +dev-checkout: ## checkout the dev branch + @branch=$(shell echo $${branch:-"dev"}) && \ + git show-branch --list | grep -q $${branch} && \ + git checkout $${branch} + +dev-checkout-upstream: ## create and checkout the dev branch, and set the upstream + @branch=$(shell echo $${branch:-"dev"}) && \ + git checkout -B $${branch} && \ + git push --set-upstream origin $${branch} || true + +main-checkout: ## checkout the main branch + @git checkout main + +##@ Utilities + +large-files: ## show the 20 largest files in the repo + @find . -printf '%s %p\n'| sort -nr | head -20 + +disk-usage: ## show the disk usage of the repo + @du -h -d 2 . + +git-sizer: ## run git-sizer + @git-sizer --verbose + +gc-prune: ## garbage collect and prune + @git gc --prune=now + +##@ Setup + +install-node: ## install node + @export NVM_DIR="$${HOME}/.nvm"; \ + [ -s "$${NVM_DIR}/nvm.sh" ] && . "$${NVM_DIR}/nvm.sh"; \ + nvm install --lts + +nvm-ls: ## list node versions + @export NVM_DIR="$${HOME}/.nvm"; \ + [ -s "$${NVM_DIR}/nvm.sh" ] && . "$${NVM_DIR}/nvm.sh"; \ + nvm ls + +set-default-node: ## set default node + @export NVM_DIR="$${HOME}/.nvm"; \ + [ -s "$${NVM_DIR}/nvm.sh" ] && . "$${NVM_DIR}/nvm.sh"; \ + nvm alias default node + +install-pipx: ## install pipx (pre-requisite for external tools) + @command -v pipx &> /dev/null || pip install --user pipx || true + +install-copier: install-pipx ## install copier (pre-requisite for init-project) + @command -v copier &> /dev/null || pipx install copier || true + +install-poetry: install-pipx ## install poetry (pre-requisite for install) + @command -v poetry &> /dev/null || pipx install poetry || true + +install-poe: install-pipx ## install poetry (pre-requisite for install) + @command -v poe &> /dev/null || pipx install poethepoet || true + +install-commitzen: install-pipx ## install commitzen (pre-requisite for commit) + @command -v cz &> /dev/null || pipx install commitizen || true + +install-precommit: install-pipx ## install pre-commit + @command -v pre-commit &> /dev/null || pipx install pre-commit || true + +install-precommit-hooks: install-precommit ## install pre-commit hooks + @pre-commit install + +initialize: install-pipx ## initialize the project environment + @pipx install copier + @pipx install poethepoet + @pipx install commitizen + @pipx install pre-commit + @pre-commit install + +init-project: initialize ## initialize the project (Warning: do this only once!) + @copier --answers-file .copier-config.yaml --vcs-ref=HEAD gh:entelecheia/hyperfast-python-template . + +reinit-project: install-copier ## reinitialize the project (Warning: this may overwrite existing files!) + @bash -c 'args=(); while IFS= read -r file; do args+=("--skip" "$$file"); done < .copierignore; copier "$${args[@]}" --answers-file .copier-config.yaml --vcs-ref=HEAD . .' + +reinit-project-force: install-copier ## initialize the project ignoring existing files (Warning: this will overwrite existing files!) + @bash -c 'args=(); while IFS= read -r file; do args+=("--skip" "$$file"); done < .copierignore; copier "$${args[@]}" --answers-file .copier-config.yaml --force --vcs-ref=HEAD . .' diff --git a/.copier-template/README.md b/.copier-template/README.md new file mode 100644 index 0000000..57c8a80 --- /dev/null +++ b/.copier-template/README.md @@ -0,0 +1,66 @@ +# Hyperfast Docker Template + +This repository provides Docker setup instructions to install and run Stable-Diffusion-WebUI. It uses NVIDIA's PyTorch container as a base image and includes several additional packages necessary for Stable-Diffusion-WebUI. + +## Prerequisites + +- Docker +- Docker Compose +- NVIDIA Docker (for GPU support) + +## Setup + +1. Clone the repository and navigate to the directory. + +2. Export the environment variables and build the Docker image: + + ``` + set -a; source .docker/docker.env; set +a; docker-compose --project-directory . -f .docker/docker-compose.yaml build + ``` + + The `docker.env` file includes various configuration options and environment variables. Feel free to adjust them to suit your needs. For instance, you may want to change the default NVIDIA Docker image in the `BUILD_FROM` variable, or the port mapping in `SDW_HOST_SSH_PORT` and `SDW_HOST_GRADIO_PORT`. + + The `set -a; source .docker/docker.env; set +a;` sequence of commands exports the environment variables in the `.docker/docker.env` file to the shell environment, making them available to the subsequent `docker-compose` command. This method allows us to use shell commands in the variable definitions, like `"$(whoami)"` for the `USERNAME` variable. We're using this approach instead of the `--env-file` argument because we need these environment variables to be available to the `docker-compose` command itself, not just to the services defined in the `docker-compose.yml` file. + +3. Start the Docker container: + + ``` + set -a; source .docker/docker.env; set +a; docker-compose --project-directory . -f .docker/docker-compose.yaml up + ``` + + This will start a Docker container with the image built in the previous step. The container will run a bash command (`webui.sh`) specified in the `command` section of the `docker-compose.yml` file. + +## Usage + +After starting the container, you can access the Stable-Diffusion-WebUI at `localhost:`. By default, the port is set to `18860`. + +You can also SSH into the container using the SSH port specified in `SDW_HOST_SSH_PORT`. By default, the port is set to `2722`. + +## Volumes + +The `docker-compose.yml` file specifies several volumes that bind mount directories on the host to directories in the container. These include the Hugging Face cache and data directories, the workspace directory, and a scripts directory. Changes made in these directories will persist across container restarts. + +## Troubleshooting + +If you encounter any issues with this setup, please check the following: + +- Make sure Docker and Docker Compose are installed correctly. +- Make sure NVIDIA Docker is installed if you're planning to use GPU acceleration. +- Ensure the environment variables in the `docker.env` file are correctly set. +- Check the Docker and Docker Compose logs for any error messages. + +## Environment Variables + +The `set -a; source .docker/docker.env; set +a;` sequence of commands in the setup steps exports the environment variables in the `.docker/docker.env` file to the shell environment, making them available to the subsequent `docker-compose` command. + +In Docker, environment variables can be used in the `docker-compose.yml` file to customize the behavior of the services. This is a common practice when you want to set different configurations for development, testing, and production environments. + +The `docker-compose` command has an `--env-file` argument, but it's used to set the environment variables for the services defined in the `docker-compose.yml` file, not for the `docker-compose` command itself. The variables defined in the `--env-file` will not be substituted into the `docker-compose.yml` file. + +However, the environment variables we set in the `.docker/docker.env` file are used in the `docker-compose.yml` file. For example, the `$BUILD_FROM` variable is used to set the base image for the Docker build. Therefore, we need to export these variables to the shell environment before running the `docker-compose` command. + +This method also allows us to use shell commands in the variable definitions, like `"$(whoami)"` for the `USERNAME` variable, which wouldn't be possible if we used the `--env-file` argument. Shell commands in the `.env` file are not evaluated. + +## Contributing + +Contributions to improve this setup are welcome. Please feel free to submit a pull request or open an issue on the GitHub repository. diff --git a/.copier-template/{{ _copier_conf.answers_file }}.jinja b/.copier-template/{{ _copier_conf.answers_file }}.jinja new file mode 100644 index 0000000..d8cdb6a --- /dev/null +++ b/.copier-template/{{ _copier_conf.answers_file }}.jinja @@ -0,0 +1,2 @@ +# Changes here will be overwritten by Copier; do NOT edit manually +{{ _copier_answers|to_nice_yaml }} diff --git a/.copierignore b/.copierignore new file mode 100644 index 0000000..9bb30ca --- /dev/null +++ b/.copierignore @@ -0,0 +1,4 @@ +.copierignore +.tasks-extra.toml +CHANGELOG.md +pyproject.toml diff --git a/.coveragerc b/.coveragerc new file mode 100644 index 0000000..0966f9c --- /dev/null +++ b/.coveragerc @@ -0,0 +1,2 @@ +[html] +directory = tests/htmlcov diff --git a/.docker/.docker-scripts/docker-build.sh b/.docker/.docker-scripts/docker-build.sh new file mode 100644 index 0000000..0038200 --- /dev/null +++ b/.docker/.docker-scripts/docker-build.sh @@ -0,0 +1,3 @@ +#!/bin/bash +# add your custom commands here that should be executed when building the docker image +echo "Building docker image..." diff --git a/.docker/.docker-scripts/docker-init-dotfiles.sh b/.docker/.docker-scripts/docker-init-dotfiles.sh new file mode 100644 index 0000000..a5e8d9d --- /dev/null +++ b/.docker/.docker-scripts/docker-init-dotfiles.sh @@ -0,0 +1,5 @@ +#!/bin/bash +# this script should be executed only once when the docker container is created +chezmoi update --apply=false +chezmoi init --apply --verbose +dotu diff --git a/.docker/.docker-scripts/docker-run.sh b/.docker/.docker-scripts/docker-run.sh new file mode 100644 index 0000000..337baba --- /dev/null +++ b/.docker/.docker-scripts/docker-run.sh @@ -0,0 +1,14 @@ +#!/bin/bash +# add your custom commands here that should be executed every time the docker container starts +echo "Starting docker container..." + +# start ssh server +sudo service ssh start +# start jupyter lab +jupyter lab \ + --no-browser \ + --notebook-dir="$WORKSPACE_ROOT" \ + --ServerApp.token="$JUPYTER_TOKEN" \ + --port="$JUPYTER_PORT" \ + --ip=0.0.0.0 \ + --allow-root diff --git a/.docker/.docker-scripts/requirements.txt b/.docker/.docker-scripts/requirements.txt new file mode 100644 index 0000000..e69de29 diff --git a/.docker/.dockerignore b/.docker/.dockerignore new file mode 100644 index 0000000..4b8fbe5 --- /dev/null +++ b/.docker/.dockerignore @@ -0,0 +1,17 @@ +.virtual_documents +jupyterhub* +.vscode +.history +workspace +notebooks +.workspace +data +refs +tmp +_tmp_ +logs +outputs +runs +.env* +!.env.project +!.env.vault diff --git a/.docker/Dockerfile b/.docker/Dockerfile new file mode 100644 index 0000000..b020bb0 --- /dev/null +++ b/.docker/Dockerfile @@ -0,0 +1,85 @@ +ARG ARG_BUILD_FROM="nvcr.io/nvidia/pytorch:23.04-py3" +FROM $ARG_BUILD_FROM + +ARG DEBIAN_FRONTEND=noninteractive +ENV TZ=Asia/Seoul + +RUN apt-get update && apt-get upgrade -y \ + && apt-get install -y fontconfig fonts-nanum curl git tzdata locales locales-all \ + libcairo2-dev imagemagick ffmpeg sudo software-properties-common openssh-server \ + && add-apt-repository ppa:git-core/ppa -y \ + && apt-get update \ + && apt-get -y install --no-install-recommends git \ + && apt-get autoremove -y \ + && apt-get clean -y + +# Set up locale +RUN localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || true +# fix sudo setrlimit(RLIMIT_CORE) error +RUN echo "Set disable_coredump false" > /etc/sudo.conf + +# Add a non-root user to a container +ARG ARG_USERNAME="dev" +ARG ARG_USER_UID=1000 +ARG ARG_USER_GID=$ARG_USER_UID +ARG ARG_USER_FULLNAME="Dev User" +ARG ARG_USER_EMAIL="dev@domain.com" +ARG ARG_GITHUB_USERNAME="" +ARG ARG_SYSTEM_HOSTNAME="dev-container" +ARG ARG_WORKSPACE_LOCATION="/" +ARG ARG_WORKSPACE_ROOT="/workspace" + +ENV USERNAME $ARG_USERNAME +ENV USER_UID $ARG_USER_UID +ENV USER_GID $ARG_USER_GID +ENV USER_FULLNAME $ARG_USER_FULLNAME +ENV USER_EMAIL $ARG_USER_EMAIL +ENV GITHUB_USERNAME $ARG_GITHUB_USERNAME +ENV SYSTEM_HOSTNAME $ARG_SYSTEM_HOSTNAME +ENV WORKSPACE_LOCATION $ARG_WORKSPACE_LOCATION +ENV WORKSPACE_ROOT $ARG_WORKSPACE_ROOT +# print all args to check +RUN echo "USERNAME: $USERNAME from $ARG_USERNAME" +RUN echo "USER_UID: $USER_UID from $ARG_USER_UID" +RUN echo "USER_GID: $USER_GID from $ARG_USER_GID" +RUN echo "USER_FULLNAME: $USER_FULLNAME from $ARG_USER_FULLNAME" +RUN echo "USER_EMAIL: $USER_EMAIL from $ARG_USER_EMAIL" +RUN echo "GITHUB_USERNAME: $GITHUB_USERNAME from $ARG_GITHUB_USERNAME" +RUN echo "SYSTEM_HOSTNAME: $SYSTEM_HOSTNAME from $ARG_SYSTEM_HOSTNAME" +RUN echo "WORKSPACE_LOCATION: $WORKSPACE_LOCATION from $ARG_WORKSPACE_LOCATION" +RUN echo "WORKSPACE_ROOT: $WORKSPACE_ROOT from $ARG_WORKSPACE_ROOT" + +# Create the user +RUN groupadd --gid $USER_GID $USERNAME \ + && adduser --uid $USER_UID --gid $USER_GID --force-badname --disabled-password --gecos "" $USERNAME \ + && echo "$USERNAME:$USERNAME" | chpasswd \ + && adduser $USERNAME sudo \ + && echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +# install dotfiles +ENV DOTFILES_APPLY_ROOTMOI=0 +ENV DOTFILES_USE_CODE=1 +ENV DOTFILES_USE_PYTHON_TOOLS=1 +ENV REMOTE_CONTAINERS=1 +# ENV DOTFILES_DEBUG=1 + +ENV PIP_DEFAULT_TIMEOUT 100 +ENV DS_BUILD_FUSED_ADAM 1 +ENV PYTHONDONTWRITEBYTECODE 1 +ENV PYTHONUNBUFFERED 1 +ENV KMP_DUPLICATE_LIB_OK TRUE + +USER $USERNAME +RUN sudo rm -rf $WORKSPACE_ROOT && sudo mkdir -p $WORKSPACE_ROOT +RUN sudo chown -R $USERNAME:$USERNAME $WORKSPACE_ROOT +RUN sh -c "$(wget -qO- https://dotfiles.entelecheia.ai/install)" + +# set workdir to user home +WORKDIR /home/$USERNAME +COPY ./.docker/scripts/ ./scripts/ +RUN sudo chmod 755 ./scripts/devenv-entrypoint.sh + +WORKDIR $WORKSPACE_ROOT +CMD ["zsh"] diff --git a/.docker/Dockerfile.app b/.docker/Dockerfile.app new file mode 100644 index 0000000..0cbf17b --- /dev/null +++ b/.docker/Dockerfile.app @@ -0,0 +1,87 @@ +# Sets the base image for subsequent instructions +ARG ARG_BUILD_FROM="nvcr.io/nvidia/pytorch:23.02-py3" +FROM $ARG_BUILD_FROM + +# Setting this argument prevents interactive prompts during the build process +ARG DEBIAN_FRONTEND=noninteractive +# Updates the image and installs necessary packages +RUN apt-get update \ + && apt-get install -y curl sudo \ + fontconfig fonts-nanum tzdata locales locales-all \ + google-perftools libcairo2-dev imagemagick ffmpeg \ + software-properties-common openssh-server \ + # Adds PPA for the latest git version + && add-apt-repository ppa:git-core/ppa -y \ + && apt-get update \ + && apt-get -y install --no-install-recommends git \ + # Cleans up unnecessary packages to reduce image size + && apt-get autoremove -y \ + && apt-get clean -y + +# Sets up the locale to en_US.UTF-8 +RUN localedef -v -c -i en_US -f UTF-8 en_US.UTF-8 || true +# Fixes sudo error related to core dumps +RUN echo "Set disable_coredump false" > /etc/sudo.conf + +# Setting ARGs and ENVs for user creation and workspace setup +ARG ARG_USERNAME="app" +ARG ARG_USER_UID=1000 +ARG ARG_USER_GID=$ARG_USER_UID +ARG ARG_WORKSPACE_ROOT="/workspace" +ENV USERNAME $ARG_USERNAME +ENV USER_UID $ARG_USER_UID +ENV USER_GID $ARG_USER_GID +ENV WORKSPACE_ROOT $ARG_WORKSPACE_ROOT + +# Creates a non-root user with sudo privileges +RUN groupadd --gid $USER_GID $USERNAME \ + && adduser --uid $USER_UID --gid $USER_GID --force-badname --disabled-password --gecos "" $USERNAME \ + && echo "$USERNAME:$USERNAME" | chpasswd \ + && adduser $USERNAME sudo \ + && echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers \ + && echo "$USERNAME ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/$USERNAME \ + && chmod 0440 /etc/sudoers.d/$USERNAME + +# Switches to the newly created user +USER $USERNAME +# Sets up the workspace for the user +RUN sudo rm -rf $WORKSPACE_ROOT && sudo mkdir -p $WORKSPACE_ROOT/projects +RUN sudo chown -R $USERNAME:$USERNAME $WORKSPACE_ROOT + +# Installs Python packages required for the build +RUN sudo pip install -U pip setuptools + +# Setting ARGs and ENVs for Stable-Diffusion-WebUI GitHub repository +ARG ARG_APP_GITHUB_USERNAME="entelecheia" +ARG ARG_APP_GITHUB_REPO="entelecheia" +ARG ARG_APP_INSTALL_ROOT="/workspace/projects" +ARG ARG_APP_CLONE_DIR=$ARG_APP_GITHUB_REPO +ENV APP_GITHUB_USERNAME $ARG_APP_GITHUB_USERNAME +ENV APP_GITHUB_REPO $ARG_APP_GITHUB_REPO +ENV APP_INSTALL_ROOT $ARG_APP_INSTALL_ROOT +ENV APP_CLONE_DIR $ARG_APP_CLONE_DIR + +# Sets the working directory to workspace root +WORKDIR $WORKSPACE_ROOT +# Copies scripts from host into the image +COPY ./.docker/scripts/ ./scripts/ +# Installs Python dependencies listed in requirements.txt +RUN pip install -r ./scripts/requirements.txt + +# Clones the stable-diffusion-webui repository from GitHub +RUN git clone "https://github.com/$APP_GITHUB_USERNAME/$APP_GITHUB_REPO.git" $APP_INSTALL_ROOT/$APP_CLONE_DIR + +# Changes the working directory to the cloned repository +WORKDIR $APP_INSTALL_ROOT/$APP_CLONE_DIR + +# Sets the time zone within the container +ENV TZ=Asia/Seoul + +# Setting ARGs and ENVs for GitHub branch and server name +ARG ARG_APP_GITHUB_BRANCH="docker-run" +ARG ARG_APP_SERVER_NAME +ENV APP_GITHUB_BRANCH $ARG_APP_GITHUB_BRANCH +ENV APP_SERVER_NAME $ARG_APP_SERVER_NAME + +# Specifies the command that will be executed when the container is run +CMD ["bash"] diff --git a/.docker/docker-compose-app.yaml b/.docker/docker-compose-app.yaml new file mode 100644 index 0000000..cb1fe96 --- /dev/null +++ b/.docker/docker-compose-app.yaml @@ -0,0 +1,60 @@ +version: "3" + +services: + # Defines a service named "workspace" + workspace: + build: + # Sets the build context to the current directory + context: . + # Specifies the Dockerfile to use for the build + dockerfile: .docker/Dockerfile + # Specifies build-time variables (ARGs) + args: + ARG_BUILD_FROM: $BUILD_FROM + ARG_USERNAME: $USERNAME + ARG_USER_UID: $USER_UID + ARG_USER_GID: $USER_GID + ARG_WORKSPACE_ROOT: $CONTAINER_WORKSPACE_ROOT + ARG_APP_GITHUB_USERNAME: $APP_GITHUB_USERNAME + ARG_APP_GITHUB_REPO: $APP_GITHUB_REPO + ARG_APP_INSTALL_ROOT: $APP_INSTALL_ROOT + ARG_APP_CLONE_DIR: $APP_CLONE_DIR + ARG_APP_GITHUB_BRANCH: $APP_GITHUB_BRANCH + ARG_APP_SERVER_NAME: $APP_SERVER_NAME + # Sets the image name for the built image + image: $IMAGE_NAME + # Sets the hostname of the container + hostname: $CONTAINER_HOSTNAME + command: + # Specifies the command to be executed when the container is run + - bash + - $CONTAINER_WORKSPACE_ROOT/scripts/launch.sh + ulimits: + # Sets the stack size and memory lock limits + stack: 67108864 + memlock: -1 + # Allows the container to use the host's IPC namespace + ipc: host + ports: + # Maps the container's SSH and Web service ports to the host's ports + - "$HOST_SSH_PORT:$SSH_PORT" + - "$HOST_WEB_SVC_PORT:$WEB_SVC_PORT" + volumes: + # Maps directories from the host to the container + - "$HOME/.cache:/home/$USERNAME/.cache" + - "$HOME/.huggingface:/home/$USERNAME/.huggingface" + - "$WORKSPACE_ROOT:$CONTAINER_WORKSPACE_ROOT" + - "$PWD/.docker/scripts:$CONTAINER_WORKSPACE_ROOT/scripts" + deploy: + resources: + reservations: + devices: + # Reserves the specified GPU for the container + - driver: nvidia + device_ids: ["${CUDA_DEVICE_ID}"] + capabilities: [gpu] +networks: + default: + # Sets the name of the default network and makes it external + name: $DOCKER_NETWORK_NAME + external: true diff --git a/.docker/docker-compose.yaml b/.docker/docker-compose.yaml new file mode 100644 index 0000000..b9d1078 --- /dev/null +++ b/.docker/docker-compose.yaml @@ -0,0 +1,28 @@ +version: "3" + +services: + workspace: + build: + context: . + dockerfile: .docker/Dockerfile + args: + ARG_BUILD_FROM: $BUILD_FROM + image: $IMAGE_NAME + hostname: $CONTAINER_HOSTNAME + command: + - bash + ulimits: + stack: 67108864 + memlock: -1 + ipc: host + deploy: + resources: + reservations: + devices: + - driver: nvidia + device_ids: ["${CUDA_DEVICE_ID}"] + capabilities: [gpu] +networks: + default: + name: $DOCKER_NETWORK_NAME + external: true diff --git a/.docker/docker.app.env b/.docker/docker.app.env new file mode 100644 index 0000000..4289f17 --- /dev/null +++ b/.docker/docker.app.env @@ -0,0 +1,46 @@ +######################################################### +# Configuration parameters for the docker project # +# Change the variables below to your need: # +######################################################### +APP_NAME: "hyperfast-docker" # The name of the project +APP_VERSION="1.0.0" # The version of the project +APP_VARIANT="base" # The variant of the project +APP_GITHUB_USERNAME="entelecheia" # The GitHub username of the project repository +APP_GITHUB_REPO="entelecheia" # The GitHub repository name of the project +APP_GITHUB_BRANCH="docker-run" # The branch of the project to clone +APP_INSTALL_ROOT="/workspace/projects" # The directory in the container where the project will be installed +APP_CLONE_DIR=${APP_GITHUB_REPO} # The directory name for the cloned project +APP_SERVER_NAME= # The server name (optional, can be left empty) +APP_CUDA_DEVICE_ID="0" # The ID of the CUDA device to use +APP_HOST_SSH_PORT="2020" # The SSH port on the host machine to be mapped to the container's SSH port +APP_HOST_WEB_SVC_PORT="18880" # The Web service port on the host machine to be mapped to the container's Web service port + +####################################################################################### +# Please do not make any changes below this line if you don't know what you are doing # +# change the variables above to your need # +####################################################################################### +# docker build: Configuration parameters for building the Docker image +DOCKER_NAME=${APP_NAME:-"hyperfast-docker"} # The base name of the Docker image. If not set, "sd-webui" will be used. +VERSION=${APP_VERSION:-"1.0.0"} # The version of the Docker image. If not set, "v1.2.0" will be used. +VARIANT=${APP_VARIANT:-"base"} # The variant of the Docker image. If not set, "runtime" will be used. +USERNAME=${USERNAME:-"$(whoami)"} # The username in the Docker container. If not set, the current user's username will be used. +DOCKER_USERNAME=${DOCKER_USERNAME:-"$(whoami)"} # The username for Docker. If not set, the current user's username will be used. +USER_UID=${USER_UID:-"$(id -u)"} # The user ID in the Docker container. If not set, the current user's ID will be used. +USER_GID=${USER_GID:-"$(id -g)"} # The group ID in the Docker container. If not set, the current user's group ID will be used. +SYSTEM_HOSTNAME=${SYSTEM_HOSTNAME:-"$(hostname)"} # The hostname of the Docker container. If not set, the system's hostname will be used. + +IMAGE_TAG="${VERSION}-${VARIANT}" # The tag of the Docker image +IMAGE_NAME="${DOCKER_USERNAME}/${DOCKER_NAME}:${IMAGE_TAG}" # The full name of the Docker image +BUILD_FROM="nvcr.io/nvidia/pytorch:23.04-py3" # The base image for the Docker build + +WORKSPACE_ROOT="/home/${USERNAME}/workspace" # The workspace directory on the host machine +CONTAINER_WORKSPACE_ROOT="/workspace" # The workspace directory in the Docker container +CONTAINER_HOSTNAME="${DOCKER_NAME}-${SYSTEM_HOSTNAME}" # The hostname of the Docker container + +# docker run: Configuration parameters for running the Docker container +DOCKER_NETWORK_NAME="${DOCKER_NAME}-network" # The name of the Docker network +CUDA_DEVICE_ID=${APP_CUDA_DEVICE_ID} # The ID of the CUDA device to use. If not set, device 0 will be used. +SSH_PORT="22" # The SSH port in the Docker container +HOST_SSH_PORT=${APP_HOST_SSH_PORT} # The SSH port on the host machine to be mapped to the container's SSH port. If not set, port 2722 will be used. +WEB_SVC_PORT="8000" # The Web service port in the Docker container +HOST_WEB_SVC_PORT=${APP_HOST_WEB_SVC_PORT} # The Web service port on the host machine to be mapped to the container's Web service port. If not set, port 18860 will be used. diff --git a/.docker/docker.env b/.docker/docker.env new file mode 100644 index 0000000..1e350d9 --- /dev/null +++ b/.docker/docker.env @@ -0,0 +1,13 @@ +# for docker build +DOCKER_NAME=${DOCKER_NAME:-"devenv-base"} +VERSION=${VERSION:-"23.04-py3"} +VARIANT=${VARIANT:-"dotfiles"} + +IMAGE_TAG="${VERSION}-${VARIANT}" +IMAGE_NAME="${DOCKER_USERNAME}/${DOCKER_NAME}:${IMAGE_TAG}" +BUILD_FROM="nvcr.io/nvidia/pytorch:${VERSION}" +CONTAINER_HOSTNAME="${DOCKER_NAME}-${SYSTEM_HOSTNAME}" + +# for docker run +DOCKER_NETWORK_NAME="${DOCKER_NAME}-network" +CUDA_DEVICE_ID=${CUDA_DEVICE_ID:-"0"} diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..04a3f23 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,24 @@ +# Check http://editorconfig.org for more information + +# top-most .editorconfig file +root = true + +# Unix-style newlines with a newline ending every file +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 +trim_trailing_whitespace = true +indent_style = space + +[*.py] +profile = black + +[Makefile] +indent_style = tab + +[*.md] +trim_trailing_whitespace = false + +[*.{yaml,yml}] +indent_size = 2 diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..6313b56 --- /dev/null +++ b/.gitattributes @@ -0,0 +1 @@ +* text=auto eol=lf diff --git a/.github/dependabot.yaml b/.github/dependabot.yaml new file mode 100644 index 0000000..ee28f46 --- /dev/null +++ b/.github/dependabot.yaml @@ -0,0 +1,27 @@ +# To get started with Dependabot version updates, you'll need to specify which +# package ecosystems to update and where the package manifests are located. +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + time: "03:00" + timezone: "Asia/Seoul" + labels: + - ":game_die: dependencies" + - ":robot: bot" + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" + time: "03:00" + timezone: "Asia/Seoul" + allow: + - dependency-type: "production" + labels: + - ":game_die: dependencies" + - ":robot: bot" diff --git a/.github/labels.yaml b/.github/labels.yaml new file mode 100644 index 0000000..f7f83aa --- /dev/null +++ b/.github/labels.yaml @@ -0,0 +1,66 @@ +--- +# Labels names are important as they are used by Release Drafter to decide +# regarding where to record them in changelog or if to skip them. +# +# The repository labels will be automatically configured using this file and +# the GitHub Action https://github.com/marketplace/actions/github-labeler. +- name: breaking + description: Breaking Changes + color: bfd4f2 +- name: bug + description: Something isn't working + color: d73a4a +- name: build + description: Build System and Dependencies + color: bfdadc +- name: ci + description: Continuous Integration + color: 4a97d6 +- name: dependencies + description: Pull requests that update a dependency file + color: 0366d6 +- name: documentation + description: Improvements or additions to documentation + color: 0075ca +- name: duplicate + description: This issue or pull request already exists + color: cfd3d7 +- name: enhancement + description: New feature or request + color: a2eeef +- name: github_actions + description: Pull requests that update Github_actions code + color: "000000" +- name: good first issue + description: Good for newcomers + color: 7057ff +- name: help wanted + description: Extra attention is needed + color: 008672 +- name: invalid + description: This doesn't seem right + color: e4e669 +- name: performance + description: Performance + color: "016175" +- name: python + description: Pull requests that update Python code + color: 2b67c6 +- name: question + description: Further information is requested + color: d876e3 +- name: refactoring + description: Refactoring + color: ef67c4 +- name: removal + description: Removals and Deprecations + color: 9ae7ea +- name: style + description: Style + color: c120e5 +- name: testing + description: Testing + color: b1fc6f +- name: wontfix + description: This will not be worked on + color: ffffff diff --git a/.github/release.yaml b/.github/release.yaml new file mode 100644 index 0000000..bdf1094 --- /dev/null +++ b/.github/release.yaml @@ -0,0 +1,20 @@ +--- +changelog: + exclude: + labels: + - ignore-for-release + authors: + - octocat + categories: + - title: Breaking Changes 🛠 + labels: + - breaking + - title: Exciting New Features 🎉 + labels: + - feature + - title: Fixes 🔧 + labels: + - fix + - title: Other Changes + labels: + - "*" diff --git a/.github/workflows/deploy-docs.yaml b/.github/workflows/deploy-docs.yaml new file mode 100644 index 0000000..6435b4a --- /dev/null +++ b/.github/workflows/deploy-docs.yaml @@ -0,0 +1,35 @@ +--- +name: deploy-docs +on: + workflow_call: + workflow_dispatch: + push: + branches: + - main + # Paths can be used to only trigger actions when you have edited certain files, such as a file within the /docs directory + paths: + - "README.md" + - "mkdocs.yaml" + - "docs/**/*" + - ".github/workflows/deploy-docs.yaml" + +permissions: + contents: write + +jobs: + deploy: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - uses: actions/setup-python@v4 + with: + python-version: 3.x + + - uses: actions/cache@v3 + with: + key: ${{ github.ref }} + path: .cache + - run: | + pip install -r docs/requirements.txt + mkdocs gh-deploy --force diff --git a/.github/workflows/prerelease.yaml b/.github/workflows/prerelease.yaml new file mode 100644 index 0000000..6584e6c --- /dev/null +++ b/.github/workflows/prerelease.yaml @@ -0,0 +1,53 @@ +--- +name: semantic-prerelease + +on: + workflow_call: + workflow_dispatch: + push: + branches: + - pre* + - beta* + - alpha* + - rc* + - nigtly* + paths: + - 'src/**' + - 'pyproject.toml' + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + build: + # Name the Job + name: prerelease + # Set the agent to run on + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.8] + + # Load all steps # + steps: + # Checkout the code base # + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # set env variable + - name: Set env variable + run: echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + + # semantic release + - name: Python Semantic Release to PyPI + uses: relekang/python-semantic-release@master + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + repository_username: __token__ + repository_password: ${{ secrets.PYPI_API_TOKEN }} + additional_options: --prerelease --verbosity=DEBUG --define=branch=${{ env.BRANCH }} diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 0000000..92ac090 --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,56 @@ +--- +name: semantic-release + +on: + workflow_call: + workflow_dispatch: + push: + branches: + - main + paths: + - ".docker/**" + - ".copier-template/**" + +permissions: + contents: write + issues: write + pull-requests: write + +jobs: + build: + # Name the Job + name: semantic-release + # Set the agent to run on + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-latest] + python-version: [3.8] + + # Load all steps # + steps: + # Checkout the code base # + - name: Checkout Code + uses: actions/checkout@v3 + with: + fetch-depth: 0 + + # set env variable + - name: Set env variable + run: echo "BRANCH=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + + # install poetry + - name: Install poetry + run: pipx install poetry>=1.3.2 + + # install dependencies + - name: Install dependencies + run: poetry install --with dev + + - name: Python Semantic Release to PyPI + run: | + git config --global user.name "GitHub Actions" + git config --global user.email "actions@github.com" + poetry run semantic-release publish --verbosity=DEBUG --define=branch=${{ env.BRANCH }} + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..053d436 --- /dev/null +++ b/.gitignore @@ -0,0 +1,29 @@ +# hypothesis: https://github.com/HypothesisWorks/hypothesis +.hypothesis/ + +# Ignore node_modules directory +node_modules/ + +# Jupyter Notebook +_build/ + +# Python +*.py[cod] +__pycache__/ +dist/ +.tox/ +*.egg-info/ +*.egg +pytest_cache/ +pytest.xml +.mypy_cache/ + +# Coverage +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +htmlcov/ +pytest-coverage.* diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..7ed3ccb --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,55 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +default_language_version: + python: python3 +default_install_hook_types: [commit-msg, pre-commit] +default_stages: [commit, manual] +fail_fast: true +minimum_pre_commit_version: "1.4.3" +repos: + - repo: meta + hooks: + - id: check-useless-excludes + - repo: https://github.com/pre-commit/pygrep-hooks + rev: v1.10.0 + hooks: + - id: python-check-blanket-noqa + - id: python-check-blanket-type-ignore + - id: python-check-mock-methods + - id: python-no-eval + - id: python-no-log-warn + - id: text-unicode-replacement-char + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + types: [python] + - id: end-of-file-fixer + types: [python] + - id: check-yaml + - id: check-json + - id: check-added-large-files + # Prevent committing inline conflict markers + - id: check-merge-conflict + args: [--assume-in-merge] + - repo: local + hooks: + - id: commitizen + name: commitizen check + entry: cz check + args: [--allow-abort, --commit-msg-file] + stages: [commit-msg] + language: system + require_serial: true + - id: black + name: black + entry: black + require_serial: true + language: system + types: [python] + # Prevent committing .rej files + - id: forbidden-files + name: forbidden files + entry: found Copier update rejection files; review them and remove them + language: fail + files: "\\.rej$" \ No newline at end of file diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..749c2b6 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,14 @@ +# .readthedocs.yaml +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details + +# Required +version: 2 + +mkdocs: + configuration: mkdocs.yaml + +# Optionally declare the Python requirements required to build your docs +python: + install: + - requirements: docs/requirements.txt diff --git a/.tasks-extra.toml b/.tasks-extra.toml new file mode 100644 index 0000000..e69de29 diff --git a/.tasks.toml b/.tasks.toml new file mode 100644 index 0000000..582b198 --- /dev/null +++ b/.tasks.toml @@ -0,0 +1,80 @@ +[tool.poe.tasks.clean] +cmd = "rm -rf book/_build docs/_build _site" +help = "remove documentation artifacts" + +[tool.poe.tasks.version] +cmd = "semantic-release print-version --current" +help = "print the current version" + +[tool.poe.tasks.next-version] +cmd = "semantic-release print-version --next" +help = "print the next version" + +[tool.poe.tasks.changelog] +cmd = "semantic-release changelog --released" +help = "print the changelog for the current version" + +[tool.poe.tasks.next-changelog] +cmd = "semantic-release changelog --unreleased" +help = "print the changelog for the next version" + +[tool.poe.tasks.release-noop] +cmd = "semantic-release publish --noop -v DEBUG" +help = "run a dry-run of the release process" + +[tool.poe.tasks.release-ci] +cmd = "semantic-release publish -v DEBUG -D commit_author='github-actions '" +help = "run the release process in CI" + +[tool.poe.tasks.prerelease-noop] +cmd = "semantic-release publish --noop -v DEBUG --prerelease" +help = "run a dry-run of the prerelease process" + +[tool.poe.tasks.show-branches] +cmd = "git show-branch --list" +help = "show branches" + +[tool.poe.tasks.show-tags] +cmd = "git tag --list" +help = "show tags" + +[tool.poe.tasks.show-remotes] +cmd = "git remote --verbose" +help = "show remotes" + +[tool.poe.tasks.show-refs] +cmd = "git show-ref" +help = "show refs" + +[tool.poe.tasks.branch-chekcout-upstream] +sequence = [ + { cmd = "git checkout -B ${branch}" }, + { cmd = "git push --set-upstream origin ${branch}" }, +] +help = "create a new branch and push it to origin" +args = ["branch"] + +[tool.poe.tasks.branch-chekcout] +cmd = "git checkout ${branch}" +help = "checkout a branch" +args = ["branch"] + +[tool.poe.tasks.main-checkout] +cmd = "git checkout main" +help = "checkout main" + +[tool.poe.tasks.install] +cmd = "poetry install --without dev" +help = "install dependencies" + +[tool.poe.tasks.install-dev] +cmd = "poetry install --with dev" +help = "install dev dependencies" + +[tool.poe.tasks.update] +cmd = "poetry update" +help = "update dependencies" + +[tool.poe.tasks.lock] +cmd = "poetry lock" +help = "lock dependencies" diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..61f3ce4 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1 @@ + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f7e1c06 --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,132 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +We as members, contributors, and leaders pledge to make participation in our +community a harassment-free experience for everyone, regardless of age, body +size, visible or invisible disability, ethnicity, sex characteristics, gender +identity and expression, level of experience, education, socio-economic status, +nationality, personal appearance, race, caste, color, religion, or sexual +identity and orientation. + +We pledge to act and interact in ways that contribute to an open, welcoming, +diverse, inclusive, and healthy community. + +## Our Standards + +Examples of behavior that contributes to a positive environment for our +community include: + +- Demonstrating empathy and kindness toward other people +- Being respectful of differing opinions, viewpoints, and experiences +- Giving and gracefully accepting constructive feedback +- Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +- Focusing on what is best not just for us as individuals, but for the overall + community + +Examples of unacceptable behavior include: + +- The use of sexualized language or imagery, and sexual attention or advances of + any kind +- Trolling, insulting or derogatory comments, and personal or political attacks +- Public or private harassment +- Publishing others' private information, such as a physical or email address, + without their explicit permission +- Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Enforcement Responsibilities + +Community leaders are responsible for clarifying and enforcing our standards of +acceptable behavior and will take appropriate and fair corrective action in +response to any behavior that they deem inappropriate, threatening, offensive, +or harmful. + +Community leaders have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, and will communicate reasons for moderation +decisions when appropriate. + +## Scope + +This Code of Conduct applies within all community spaces, and also applies when +an individual is officially representing the community in public spaces. +Examples of representing our community include using an official e-mail address, +posting via an official social media account, or acting as an appointed +representative at an online or offline event. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported to the community leaders responsible for enforcement at +[entelecheia@hotmail.com](mailto:entelecheia@hotmail.com). +All complaints will be reviewed and investigated promptly and fairly. + +All community leaders are obligated to respect the privacy and security of the +reporter of any incident. + +## Enforcement Guidelines + +Community leaders will follow these Community Impact Guidelines in determining +the consequences for any action they deem in violation of this Code of Conduct: + +### 1. Correction + +**Community Impact**: Use of inappropriate language or other behavior deemed +unprofessional or unwelcome in the community. + +**Consequence**: A private, written warning from community leaders, providing +clarity around the nature of the violation and an explanation of why the +behavior was inappropriate. A public apology may be requested. + +### 2. Warning + +**Community Impact**: A violation through a single incident or series of +actions. + +**Consequence**: A warning with consequences for continued behavior. No +interaction with the people involved, including unsolicited interaction with +those enforcing the Code of Conduct, for a specified period of time. This +includes avoiding interactions in community spaces as well as external channels +like social media. Violating these terms may lead to a temporary or permanent +ban. + +### 3. Temporary Ban + +**Community Impact**: A serious violation of community standards, including +sustained inappropriate behavior. + +**Consequence**: A temporary ban from any sort of interaction or public +communication with the community for a specified period of time. No public or +private interaction with the people involved, including unsolicited interaction +with those enforcing the Code of Conduct, is allowed during this period. +Violating these terms may lead to a permanent ban. + +### 4. Permanent Ban + +**Community Impact**: Demonstrating a pattern of violation of community +standards, including sustained inappropriate behavior, harassment of an +individual, or aggression toward or disparagement of classes of individuals. + +**Consequence**: A permanent ban from any sort of public interaction within the +community. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], +version 2.1, available at +[https://www.contributor-covenant.org/version/2/1/code_of_conduct.html][v2.1]. + +Community Impact Guidelines were inspired by +[Mozilla's code of conduct enforcement ladder][mozilla coc]. + +For answers to common questions about this code of conduct, see the FAQ at +[https://www.contributor-covenant.org/faq][faq]. Translations are available at +[https://www.contributor-covenant.org/translations][translations]. + +[homepage]: https://www.contributor-covenant.org +[v2.1]: https://www.contributor-covenant.org/version/2/1/code_of_conduct.html +[mozilla coc]: https://github.com/mozilla/diversity +[faq]: https://www.contributor-covenant.org/faq +[translations]: https://www.contributor-covenant.org/translations diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..03324a5 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,55 @@ +# Contributing + +Contributions are welcome, and they are greatly appreciated! Every little bit +helps, and credit will always be given. You can contribute in the ways listed below. + +## Report Bugs + +Report bugs using GitHub issues. + +If you are reporting a bug, please include: + +* Your operating system name and version. +* Any details about your local setup that might be helpful in troubleshooting. +* Detailed steps to reproduce the bug. + +## Fix Bugs + +Look through the GitHub issues for bugs. Anything tagged with "bug" and "help +wanted" is open to whoever wants to implement it. + +## Implement Features + +Look through the GitHub issues for features. Anything tagged with "enhancement" +and "help wanted" is open to whoever wants to implement it. + +## Write Documentation + +Hyperfast Docker Template could always use more documentation, whether as part of the +official Hyperfast Docker Template docs, in docstrings, or even on the web in blog posts, +articles, and such. + +## Submit Feedback + +The best way to send feedback is to file an issue on GitHub. + +If you are proposing a feature: + +* Explain in detail how it would work. +* Keep the scope as narrow as possible, to make it easier to implement. +* Remember that this is a volunteer-driven project, and that contributions + are welcome :) + +## Get Started + +Ready to contribute? Here's how to set up `Hyperfast Docker Template` for local development. + +1. Fork the repo on GitHub. +2. Clone your fork locally. +3. Create a branch for local development and make changes locally. +4. Commit your changes and push your branch to GitHub. +5. Submit a pull request through the GitHub website. + +## Code of Conduct + +Please note that the Hyperfast Docker Template project is released with a [Contributor Code of Conduct](CODE_OF_CONDUCT.md). By contributing to this project you agree to abide by its terms. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..ee0c2bd --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2023 Young Joon Lee + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0d4276b --- /dev/null +++ b/Makefile @@ -0,0 +1,109 @@ +# To do stuff with make, you type `make` in a directory that has a file called +# "Makefile". You can also type `make -f ` to use a different filename. +# +# A Makefile is a collection of rules. Each rule is a recipe to do a specific +# thing, sort of like a grunt task or an npm package.json script. +# +# A rule looks like this: +# +# : +# +# +# The "target" is required. The prerequisites are optional, and the commands +# are also optional, but you have to have one or the other. +# +# Type `make` to show the available targets and a description of each. +# +.DEFAULT_GOAL := help +.PHONY: help +help: ## Display this help + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-25s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) + + +##@ Clean-up + +clean: ## run all clean commands + @poe clean + +##@ Git Branches + +show-branches: ## show all branches + @git show-branch --list + +dev-checkout: ## checkout the dev branch + @branch=$(shell echo $${branch:-"dev"}) && \ + git show-branch --list | grep -q $${branch} && \ + git checkout $${branch} + +dev-checkout-upstream: ## create and checkout the dev branch, and set the upstream + @branch=$(shell echo $${branch:-"dev"}) && \ + git checkout -B $${branch} && \ + git push --set-upstream origin $${branch} || true + +main-checkout: ## checkout the main branch + @git checkout main + +##@ Utilities + +large-files: ## show the 20 largest files in the repo + @find . -printf '%s %p\n'| sort -nr | head -20 + +disk-usage: ## show the disk usage of the repo + @du -h -d 2 . + +git-sizer: ## run git-sizer + @git-sizer --verbose + +gc-prune: ## garbage collect and prune + @git gc --prune=now + +##@ Setup + +install-node: ## install node + @export NVM_DIR="$${HOME}/.nvm"; \ + [ -s "$${NVM_DIR}/nvm.sh" ] && . "$${NVM_DIR}/nvm.sh"; \ + nvm install --lts + +nvm-ls: ## list node versions + @export NVM_DIR="$${HOME}/.nvm"; \ + [ -s "$${NVM_DIR}/nvm.sh" ] && . "$${NVM_DIR}/nvm.sh"; \ + nvm ls + +set-default-node: ## set default node + @export NVM_DIR="$${HOME}/.nvm"; \ + [ -s "$${NVM_DIR}/nvm.sh" ] && . "$${NVM_DIR}/nvm.sh"; \ + nvm alias default node + +install-pipx: ## install pipx (pre-requisite for external tools) + @command -v pipx &> /dev/null || pip install --user pipx || true + +install-copier: install-pipx ## install copier (pre-requisite for init-project) + @command -v copier &> /dev/null || pipx install copier || true + +install-poetry: install-pipx ## install poetry (pre-requisite for install) + @command -v poetry &> /dev/null || pipx install poetry || true + +install-poe: install-pipx ## install poetry (pre-requisite for install) + @command -v poe &> /dev/null || pipx install poethepoet || true + +install-commitzen: install-pipx ## install commitzen (pre-requisite for commit) + @command -v cz &> /dev/null || pipx install commitizen || true + +install-precommit: install-pipx ## install pre-commit + @command -v pre-commit &> /dev/null || pipx install pre-commit || true + +install-precommit-hooks: install-precommit ## install pre-commit hooks + @pre-commit install + +initialize: install-pipx ## initialize the project environment + @pipx install copier + @pipx install poethepoet + @pipx install commitizen + @pipx install pre-commit + @pre-commit install + +init-project: initialize ## initialize the project (Warning: do this only once!) + @copier --answers-file .copier-config.yaml --vcs-ref=HEAD gh:entelecheia/hyperfast-docker-template . + +reinit-project: install-copier ## reinitialize the project (Warning: this may overwrite existing files!) + @bash -c 'args=(); while IFS= read -r file; do args+=("--skip" "$$file"); done < .copierignore; copier "$${args[@]}" --answers-file .copier-config.yaml --vcs-ref=HEAD gh:entelecheia/hyperfast-docker-template .' diff --git a/README.md b/README.md index a07cb7f..7a020c8 100644 --- a/README.md +++ b/README.md @@ -1 +1,108 @@ -# hyperfast-docker-template \ No newline at end of file +# Hyperfast Docker Template + +[![version-image]][release-url] +[![release-date-image]][release-url] +[![license-image]][license-url] + + + +[pypi-image]: https://img.shields.io/pypi/v/hyperfast-docker-template +[license-image]: https://img.shields.io/github/license/entelecheia/hyperfast-docker-template +[license-url]: https://github.com/entelecheia/hyperfast-docker-template/blob/main/LICENSE +[version-image]: https://img.shields.io/github/v/release/entelecheia/hyperfast-docker-template?sort=semver +[release-date-image]: https://img.shields.io/github/release-date/entelecheia/hyperfast-docker-template +[release-url]: https://github.com/entelecheia/hyperfast-docker-template/releases +[jupyter-book-image]: https://jupyterbook.org/en/stable/_images/badge.svg +[repo-url]: https://github.com/entelecheia/hyperfast-docker-template +[pypi-url]: https://pypi.org/project/hyperfast-docker-template +[docs-url]: https://hyperfast-docker.entelecheia.ai +[changelog]: https://github.com/entelecheia/hyperfast-docker-template/blob/main/CHANGELOG.md +[contributing guidelines]: https://github.com/entelecheia/hyperfast-docker-template/blob/main/CONTRIBUTING.md + + + +Hyperfast Docker Template is a powerful tool that leverages copier to streamline the creation of new Docker projects. It simplifies and accelerates Docker configurations, fostering a highly efficient and user-friendly development experience. + +- Documentation: [https://hyperfast-docker.entelecheia.ai][docs-url] +- GitHub: [https://github.com/entelecheia/hyperfast-docker-template][repo-url] + +Hyperfast Docker Template is an innovative project creation tool that merges the flexibility of Docker with the simplicity of Copier to optimize the project creation process. This tool allows developers to quickly and efficiently set up Docker projects, thereby significantly reducing the time and effort traditionally associated with this process. + +By harnessing the power of Copier, Hyperfast Docker Template provides a user-friendly interface that developers can utilize to duplicate Docker projects with minimal hassle. It eradicates the need for extensive coding knowledge, making Docker more accessible to a wider range of users. + +Furthermore, this tool employs a hyperfast approach to Docker project templating, ensuring that new projects are up and running in the quickest time possible. This speed does not compromise the quality or functionality of the Docker projects created, but instead enhances their performance, reliability, and scalability. + +Hyperfast Docker Template also ensures consistency across Docker projects, as the use of a template guarantees that all projects conform to a standardized layout and configuration. This, in turn, enhances the overall workflow, as developers can familiarize themselves with a consistent project structure and configuration, leading to increased productivity and efficiency. + +Whether you're an experienced developer or a novice just starting out, Hyperfast Docker Template is the ideal tool to fast-track your Docker project creation process, ensuring a seamless, efficient, and highly productive development experience. + +## Prerequisites + +- Docker +- Docker Compose +- NVIDIA Docker (for GPU support) + +## Setup + +1. Clone the repository and navigate to the directory. + +2. Export the environment variables and build the Docker image: + + ``` + set -a; source .docker/docker.env; set +a; docker-compose --project-directory . -f .docker/docker-compose.yaml build + ``` + + The `docker.env` file includes various configuration options and environment variables. Feel free to adjust them to suit your needs. For instance, you may want to change the default NVIDIA Docker image in the `BUILD_FROM` variable, or the port mapping in `SDW_HOST_SSH_PORT` and `SDW_HOST_GRADIO_PORT`. + + The `set -a; source .docker/docker.env; set +a;` sequence of commands exports the environment variables in the `.docker/docker.env` file to the shell environment, making them available to the subsequent `docker-compose` command. This method allows us to use shell commands in the variable definitions, like `"$(whoami)"` for the `USERNAME` variable. We're using this approach instead of the `--env-file` argument because we need these environment variables to be available to the `docker-compose` command itself, not just to the services defined in the `docker-compose.yml` file. + +3. Start the Docker container: + + ``` + set -a; source .docker/docker.env; set +a; docker-compose --project-directory . -f .docker/docker-compose.yaml up + ``` + + This will start a Docker container with the image built in the previous step. The container will run a bash command (`webui.sh`) specified in the `command` section of the `docker-compose.yml` file. + +## Usage + +After starting the container, you can access the Stable-Diffusion-WebUI at `localhost:`. By default, the port is set to `18860`. + +You can also SSH into the container using the SSH port specified in `SDW_HOST_SSH_PORT`. By default, the port is set to `2722`. + +## Volumes + +The `docker-compose.yml` file specifies several volumes that bind mount directories on the host to directories in the container. These include the Hugging Face cache and data directories, the workspace directory, and a scripts directory. Changes made in these directories will persist across container restarts. + +## Troubleshooting + +If you encounter any issues with this setup, please check the following: + +- Make sure Docker and Docker Compose are installed correctly. +- Make sure NVIDIA Docker is installed if you're planning to use GPU acceleration. +- Ensure the environment variables in the `docker.env` file are correctly set. +- Check the Docker and Docker Compose logs for any error messages. + +## Environment Variables + +The `set -a; source .docker/docker.env; set +a;` sequence of commands in the setup steps exports the environment variables in the `.docker/docker.env` file to the shell environment, making them available to the subsequent `docker-compose` command. + +In Docker, environment variables can be used in the `docker-compose.yml` file to customize the behavior of the services. This is a common practice when you want to set different configurations for development, testing, and production environments. + +The `docker-compose` command has an `--env-file` argument, but it's used to set the environment variables for the services defined in the `docker-compose.yml` file, not for the `docker-compose` command itself. The variables defined in the `--env-file` will not be substituted into the `docker-compose.yml` file. + +However, the environment variables we set in the `.docker/docker.env` file are used in the `docker-compose.yml` file. For example, the `$BUILD_FROM` variable is used to set the base image for the Docker build. Therefore, we need to export these variables to the shell environment before running the `docker-compose` command. + +This method also allows us to use shell commands in the variable definitions, like `"$(whoami)"` for the `USERNAME` variable, which wouldn't be possible if we used the `--env-file` argument. Shell commands in the `.env` file are not evaluated. + +## Changelog + +See the [CHANGELOG] for more information. + +## Contributing + +Contributions are welcome! Please see the [contributing guidelines] for more information. + +## License + +This project is released under the [MIT License][license-url]. diff --git a/copier.yaml b/copier.yaml new file mode 100644 index 0000000..59a5caf --- /dev/null +++ b/copier.yaml @@ -0,0 +1,147 @@ +_subdirectory: .copier-template + +# Configure jinja2 defaults to make syntax highlighters lives easier +_templates_suffix: .jinja +_envops: + block_end_string: "%}" + block_start_string: "{%" + comment_end_string: "#}" + comment_start_string: "{#" + keep_trailing_newline: true + variable_end_string: "}}" + variable_start_string: "{{" + +_answers_file: .copier-docker-config.yaml + +_tasks: + # Strings get executed under system's default shell + - "[ -d .git ] || git init" + - "{% if github_repo_name %}git add remote origin https://github.com/{{ github_username }}/{{ github_repo_name }}.git 2>/dev/null || echo 'origin already exists'{% else %}echo 'No GitHub repository name provided. Skipping git remote add.'{% endif %}" + +# questions +docker_name: + type: str + help: What is your docker project name? (it will be used as docker image name) + validator: >- + {% if not (docker_name | regex_search('^[a-z][a-z0-9\-]+$')) %} + docker_name must start with a letter, followed one or more letters, digits or dashes in lowercase only + {% endif %} + +friendly_name: + default: "{{ docker_name }}" + type: str + help: What is your project friendly name? (it will be used for the title of the documentation) + +project_short_description: + type: str + help: What is your project short description? + +project_description: + type: str + help: What is your project description? + +docker_username: + type: str + help: What is your Docker username? + +docker_image_version: + default: 1.0.0 + type: str + help: What is your Docker image version? + +docker_image_variant: + default: base + type: str + help: What is your Docker image variant? + +docker_build_from: + default: nvcr.io/nvidia/pytorch:23.04-py3 + type: str + help: What is your Docker image build from? + +docker_timezone: + default: Asia/Seoul + type: str + help: What is your Docker time zone? + +docker_apt_packages: + default: "fontconfig fonts-nanum curl git tzdata locales locales-all libcairo2-dev imagemagick ffmpeg sudo software-properties-common" + type: str + help: What is your Docker apt packages to install? (separated by space) + +install_dotfiles: + default: true + type: bool + help: Do you want to install dotfiles? + +workspace_root: + default: /workspace + type: str + help: What is your workspace root? + +docker_scripts_dir: + default: .docker-scripts + type: str + help: What is your docker scripts directory? (it will be copied to the docker image. You can use it to install your own packages or run your own scripts) + +pip_requirements: + default: requirements.txt + type: str + help: What is your pip requirements file to install at the docker image? (put it in the docker scripts directory) + +build_scripts: + default: docker-build.sh + type: str + help: What is your build script to run at the docker image? (put it in the docker scripts directory) + +run_scripts: + default: docker-run.sh + type: str + help: What is your run script to run when the docker image is run? (put it in the docker scripts directory) + +use_ssh: + default: true + type: bool + help: Do you want to use SSH? + +ssh_host_port: + default: 2222 + type: int + help: "What is your SSH host port to expose? (default: 2222)" + +use_jupyter: + default: true + type: bool + help: Do you want to use Jupyter? + +jupyter_host_port: + default: 8888 + type: int + help: "What is your Jupyter host port to expose? (default: 8888)" + +github_username: + type: str + help: What is your GitHub username? + validator: >- + {% if not (github_username | regex_search('^[a-z0-9][a-z0-9\-]+$')) %} + github_username must start with a letter or digit, followed one or more letters, digits or dashes in lowercase only + {% endif %} + +email: + default: "{{ github_username }}@" + type: str + help: What is your email? + validator: >- + {% if not (email | regex_search('^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$')) %} + email must be a valid email address + {% endif %} + +github_repo_name: + default: "{{ docker_name }}" + type: str + help: What is your GitHub repository name? + +use_source_code_skeleton: + default: "{{ build_and_release | lower }}" + type: bool + help: Do you want to use the source code skeleton? (it will create `src` and `tests` directories) diff --git a/docs/CNAME b/docs/CNAME new file mode 100644 index 0000000..400d647 --- /dev/null +++ b/docs/CNAME @@ -0,0 +1 @@ +hyperfast-docker.entelecheia.ai \ No newline at end of file diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..b0d5c88 --- /dev/null +++ b/docs/api.md @@ -0,0 +1 @@ +# API Reference diff --git a/docs/index.md b/docs/index.md new file mode 100644 index 0000000..9093a34 --- /dev/null +++ b/docs/index.md @@ -0,0 +1,8 @@ +{!README.md!} + +Check out the [usage](usage) section for further information. + + +!!! note + + This project is under active development. diff --git a/docs/javascripts/mathjax.js b/docs/javascripts/mathjax.js new file mode 100644 index 0000000..117b046 --- /dev/null +++ b/docs/javascripts/mathjax.js @@ -0,0 +1,16 @@ +window.MathJax = { + tex: { + inlineMath: [["\\(", "\\)"]], + displayMath: [["\\[", "\\]"]], + processEscapes: true, + processEnvironments: true, + }, + options: { + ignoreHtmlClass: ".*|", + processHtmlClass: "arithmatex", + }, +}; + +document$.subscribe(() => { + MathJax.typesetPromise(); +}); diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..5206cce --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +mkdocs +mkdocstrings[python] +markdown-include +mkdocs-material +mkdocs-material-extensions>=1.1 +pymdown-extensions>=9.9.1 diff --git a/docs/usage.md b/docs/usage.md new file mode 100644 index 0000000..8f04b05 --- /dev/null +++ b/docs/usage.md @@ -0,0 +1 @@ +# Usage diff --git a/mkdocs.yaml b/mkdocs.yaml new file mode 100644 index 0000000..4e9b363 --- /dev/null +++ b/mkdocs.yaml @@ -0,0 +1,89 @@ +site_name: "Hyperfast Docker Template" +site_url: https://hyperfast-docker.entelecheia.ai +site_author: Young Joon Lee +site_description: >- + Hyperfast Docker Template is a powerful tool that leverages copier to streamline the creation of new Docker projects. It simplifies and accelerates Docker configurations, fostering a highly efficient and user-friendly development experience. +# Repository +repo_name: entelecheia/hyperfast-docker-template +repo_url: https://github.com/entelecheia/hyperfast-docker-template +edit_uri: edit/main/docs/ + +# Copyright +copyright: Copyright © 2023 Young Joon Lee + +# Configuration +theme: + name: material + palette: + scheme: slate + features: + - navigation.expand + - navigation.indexes + - content.action.edit + - content.action.view + - content.code.annotate + - content.code.copy + # - content.tabs.link + - content.tooltips + - search.highlight + - search.share + - search.suggest + - toc.follow + favicon: https://assets.entelecheia.ai/favicon.png # favicon: Path to the favicon image + logo: https://assets.entelecheia.ai/logo.png # logo: Path to the book logo + +# Extensions +markdown_extensions: + - markdown_include.include: + base_path: . + - pymdownx.arithmatex: + generic: true + - admonition + - pymdownx.caret + - pymdownx.details + # - pymdownx.emoji: + # emoji_generator: "!!python/name:materialx.emoji.to_svg" + # emoji_index: "!!python/name:materialx.emoji.twemoji" + - pymdownx.highlight: + anchor_linenums: true + line_spans: __span + pygments_lang_class: true + - pymdownx.inlinehilite + - pymdownx.keys + - pymdownx.snippets + - pymdownx.mark + - pymdownx.smartsymbols + - pymdownx.superfences: + custom_fences: + - name: mermaid + class: mermaid + format: "!!python/name:pymdownx.superfences.fence_code_format" + - pymdownx.tabbed: + alternate_style: true + - pymdownx.tasklist: + custom_checkbox: true + - pymdownx.tilde + +# Plugins +plugins: + - search + - mkdocstrings: + handlers: + # See: https://mkdocstrings.github.io/python/usage/ + python: + options: + docstring_style: sphinx + +# Customization +extra_javascript: + - javascripts/mathjax.js + - https://polyfill.io/v3/polyfill.min.js?features=es6 + - https://cdn.jsdelivr.net/npm/mathjax@3/es5/tex-mml-chtml.js +extra: + analytics: + provider: google + property: G-0K77490FJT +# Page tree +# nav: +# - Home: index.md +# - Usage: usage.md diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9b3c245 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,41 @@ +[tool.poetry] +name = "hyperfast-docker-template" +version = "0.0.0" +description = "Hyperfast Docker Template is a powerful tool that leverages copier to streamline the creation of new Docker projects. It simplifies and accelerates Docker configurations, fostering a highly efficient and user-friendly development experience." +authors = ["Young Joon Lee "] +license = "MIT" +homepage = "https://hyperfast-docker.entelecheia.ai" +repository = "https://github.com/entelecheia/hyperfast-docker-template" +readme = "README.md" + +[tool.poetry.dependencies] +python = ">=3.8.1,<3.12" + +[tool.poetry.group.dev] +optional = true + +[tool.poetry.group.dev.dependencies] +python-semantic-release = "^7.33.1" + +[tool.poe] +include = [".tasks.toml", ".tasks-extra.toml"] + +[tool.commitizen] +name = "cz_conventional_commits" +tag_format = "v$version" + +[tool.semantic_release] +branch = "main" +version_toml = "pyproject.toml:tool.poetry.version" +version_pattern = '.docker/docker.app.env:APP_VERSION="{version}"' +version_source = "tag" +commit_version_number = true # required for version_source = "tag" +commit_subject = "chore(release): :rocket: {version} [skip ci]" +prerelease_tag = "rc" +major_on_zero = true +tag_commit = true +changelog_file = "CHANGELOG.md" +upload_to_repository = false +upload_to_release = false +hvcs = "github" # hosting version control system, gitlab is also supported +build_command = "mkdir -p dist && echo 'No build command configured' > dist/release.txt"