Skip to content

Commit

Permalink
Devcontainer work for #702, #703 (#717)
Browse files Browse the repository at this point in the history
Closes #702
Closes #703

- Adds gdb pretty printing support,
- Moves `.conda ` and `.config` folders to parent directory to share with `mrc` development.
- Uses @trxcllnt 's new devcontainer base container which adds features such as mamba, automatic conda env mounting, etc.
- introduces script to install `pytest-kafka`
- Supports running GraphViz and SID visualizations from inside devcontainer accessable to host.

Authors:
  - Christopher Harris (https://github.com/cwharris)

Approvers:
  - Michael Demoret (https://github.com/mdemoret-nv)

URL: #717
  • Loading branch information
cwharris authored Mar 8, 2023
1 parent 72437b9 commit e36419f
Show file tree
Hide file tree
Showing 14 changed files with 192 additions and 41 deletions.
16 changes: 1 addition & 15 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -13,20 +13,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

FROM nvidia/cuda:11.5.1-devel-ubuntu20.04 AS base
FROM rapidsai/devcontainers:23.04-cuda11.8-mambaforge-ubuntu22.04 AS base

RUN apt-get update && \
apt-get install --no-install-recommends --yes \
wget git-lfs gdb iputils-ping

ENV CONDA_DIR=/opt/conda

RUN wget "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" && \
bash Mambaforge-$(uname)-$(uname -m).sh -b -p ${CONDA_DIR} && \
echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> /etc/skel/.bashrc && \
echo ". ${CONDA_DIR}/etc/profile.d/conda.sh && conda activate base" >> ~/.bashrc

ENV PATH="${CONDA_DIR}/bin:${PATH}"

ENV MAMBA_NO_BANNER=1
ENV PATH="${PATH}:/workspaces/morpheus/.devcontainer/bin"
2 changes: 2 additions & 0 deletions .devcontainer/bin/dev-kafka-logs
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml"

docker compose -f $COMPOSE_FILE logs zookeeper kafka
2 changes: 2 additions & 0 deletions .devcontainer/bin/dev-kafka-produce
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml"

if [[ $# -eq 2 ]]; then
Expand Down
2 changes: 2 additions & 0 deletions .devcontainer/bin/dev-kafka-start
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

COMPOSE_FILE="${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml"

docker compose -f $COMPOSE_FILE up -d zookeeper kafka
Expand Down
2 changes: 2 additions & 0 deletions .devcontainer/bin/dev-kafka-stop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

docker compose -f ${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml stop kafka zookeeper
33 changes: 33 additions & 0 deletions .devcontainer/bin/dev-pytest-kafka-install
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

# Replicates the install instructions from the CI runner image
sudo apt update

# Install openjdk
sudo apt install -y openjdk-11-jdk

# Clone the repo into home and run setup.py develop

if [[ ! -d ~/pytest-kafka ]]; then
git clone https://gitlab.com/karolinepauls/pytest-kafka.git ~/pytest-kafka
fi;

cd ~/pytest-kafka

python setup.py develop
2 changes: 2 additions & 0 deletions .devcontainer/bin/dev-triton-start
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

args=""

if [ $# -gt 0 ] ; then
Expand Down
2 changes: 2 additions & 0 deletions .devcontainer/bin/dev-triton-stop
Original file line number Diff line number Diff line change
Expand Up @@ -14,4 +14,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.

set -e

docker compose -f ${MORPHEUS_ROOT}/.devcontainer/docker-compose.yml stop triton
79 changes: 60 additions & 19 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,29 +38,68 @@

"containerEnv": {
"HOST_MORPHEUS_ROOT": "${localWorkspaceFolder}",
"MORPHEUS_ROOT": "${containerWorkspaceFolder}"
"MORPHEUS_ROOT": "${containerWorkspaceFolder}",
"DEFAULT_CONDA_ENV": "morpheus",
"MAMBA_NO_BANNER": "1",
"CONDA_ALWAYS_YES": "true",
"VAULT_HOST": "https://vault.ops.k8s.rapids.ai",
"DISPLAY": "${localEnv:DISPLAY:-}",
"XAUTHORITY": "${localEnv:XAUTHORITY:-}",
"XDG_SESSION_TYPE": "${localEnv:XDG_SESSION_TYPE:-}",
"XDG_RUNTIME_DIR": "${localEnv:XDG_RUNTIME_DIR:-}",
"DBUS_SESSION_BUS_ADDRESS": "${localEnv:DBUS_SESSION_BUS_ADDRESS:-}"
},

"postStartCommand": "${containerWorkspaceFolder}/.devcontainer/bin/setup-morpheus-env",

"initializeCommand": [ "./.devcontainer/init.sh" ],
"initializeCommand": [ "${localWorkspaceFolder}/.devcontainer/initialize-command.sh" ],

"remoteUser": "coder",

"mounts": [
{"type":"bind", "source": "/var/run/docker.sock", "target": "/var/run/docker.sock"},
{"type":"bind", "source": "${localWorkspaceFolder}/.cache/conda", "target": "/home/coder/.conda"},
{"type":"bind", "source": "${localEnv:HOME}/.config/gh", "target": "/home/coder/.config/gh"}
{
"type": "bind",
"source": "/tmp/.X11-unix",
"target": "/tmp/.X11-unix"
},
{
"type": "bind",
"source": "${localEnv:XDG_RUNTIME_DIR}",
"target": "${localEnv:XDG_RUNTIME_DIR}"
},
{
"type": "bind",
"source": "/run/dbus/system_bus_socket",
"target": "/run/dbus/system_bus_socket"
},
{
"type":"bind",
"source": "/var/run/docker.sock",
"target": "/var/run/docker.sock"
},
{
"type": "bind",
"source": "${localWorkspaceFolder}/.cache/conda/envs",
"target": "/home/coder/.conda/envs"
},
{
"type": "bind",
"source": "${localWorkspaceFolder}/../.conda/pkgs",
"target": "/home/coder/.conda/pkgs"
},
{
"type":"bind",
"source": "${localWorkspaceFolder}/../.config", // parent folder because sister repos are sibling dirs
"target": "/home/coder/.config"
},
{
"type": "bind",
"source": "${localWorkspaceFolder}/.devcontainer/opt/morpheus",
"target": "/opt/morpheus"
},

],

"features": {
"ghcr.io/devcontainers/features/common-utils:1": {
"uid": "1000",
"gid": "1000",
"username": "coder"
},
"ghcr.io/devcontainers/features/docker-from-docker": {},
"ghcr.io/devcontainers/features/github-cli": {}
"ghcr.io/devcontainers/features/dotnet:1": {}
},

"customizations": {
Expand All @@ -74,11 +113,13 @@
"nvidia.nsight-vscode-edition",
"twxs.cmake",
"xaver.clang-format"
]
],
"settings": {
"cmake.cmakePath": "/tmp/.current-conda-env/bin/cmake",
"cmake.languageSupport.dotnetPath": "/usr/bin/dotnet",
"C_Cpp.intelliSenseEngine": "disabled",
"python.terminal.activateEnvironment": false
}
}
},
"settings": {
"C_Cpp.intelliSenseEngine": "disabled",
"python.terminal.activateEnvironment": false
}
}
6 changes: 3 additions & 3 deletions .devcontainer/init.sh → .devcontainer/initialize-command.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
docker network inspect morpheus >/dev/null 2>&1 || docker network create morpheus

# create the parent conda folder so it's found when mounting
mkdir -p ./.cache/conda
mkdir -p ../.conda

# create a github cli config directory if it does not exist so it's found when mounting
mkdir -p ~/.config/gh
# create a config directory if it does not exist so it's found when mounting
mkdir -p ../.config
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
Expand All @@ -17,8 +17,6 @@
# Ensure our ~/.config directory has the correct permissions. If ~/.config did
# not exist, and you mount ~/.config/gh from the host, then ~/.config will be
# created with root permissions which can break things
sudo mkdir -p ~/.config
sudo chown ${USER}:${USER} ~/.config

conda_env_find(){
conda env list | grep "${@}" >/dev/null 2>/dev/null
Expand All @@ -29,6 +27,6 @@ ENV_NAME=${ENV_NAME:-morpheus}
sed -ri "s/conda activate base/conda activate $ENV_NAME/g" ~/.bashrc;

if conda_env_find "${ENV_NAME}" ; \
then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \
then mamba env update --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml --prune; \
else mamba env create --name ${ENV_NAME} -f ${MORPHEUS_ROOT}/docker/conda/environments/cuda11.5_dev.yml; \
fi
23 changes: 23 additions & 0 deletions .devcontainer/opt/morpheus/bin/update-content-command.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,23 @@
#!/bin/bash
# SPDX-FileCopyrightText: Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# Ensure our ~/.config directory has the correct permissions. If ~/.config did
# not exist, and you mount ~/.config/gh from the host, then ~/.config will be
# created with root permissions which can break things

if [[ ! -f ~/.config/.gdbinit ]]; then
cp /opt/morpheus/etc/.gdbinit ~/.config/.gdbinit
fi
3 changes: 3 additions & 0 deletions .devcontainer/opt/morpheus/etc/.gdbinit
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
add-auto-load-safe-path /opt/conda

source /opt/morpheus/etc/enable_conda_libstd_pretty_print.py
55 changes: 55 additions & 0 deletions .devcontainer/opt/morpheus/etc/enable_conda_libstd_pretty_print.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
# SPDX-FileCopyrightText: Copyright (c) 2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# -*- python -*-
import os
import subprocess
import sys

import gdb

conda_env_path = os.environ.get("CONDA_PREFIX", None)

if (conda_env_path is not None):

gcc_path = os.environ.get("GCC", None)

if (gcc_path is None):
print(
"Could not find gcc from $GCC: '{}'. Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment"
.format(gcc_path))
else:
# Get the GCC version
result = subprocess.run([gcc_path, '-dumpversion'], stdout=subprocess.PIPE)
gcc_version = result.stdout.decode("utf-8").strip()

# Build the gcc python path
gcc_python_path = os.path.join(conda_env_path, "share", "gcc-{}".format(gcc_version), "python")

if (os.path.exists(gcc_python_path)):

# Add to the path for the pretty printer
sys.path.insert(0, gcc_python_path)

# Now register the pretty printers
from libstdcxx.v6 import register_libstdcxx_printers
register_libstdcxx_printers(gdb.current_objfile())

print("Loaded stdlibc++ pretty printers")
else:
print("Could not find gcc python files at: {}".format(gcc_python_path))
print(
"Ensure gxx_linux-64, gcc_linux-64, sysroot_linux-64, and gdb have been installed into the current conda environment"
)

0 comments on commit e36419f

Please sign in to comment.