Select Git revision
Dockerfile 10.74 KiB
# syntax=docker/dockerfile:1
# use scipy-notebook and install torch from PyPI after conda-forge packages
FROM quay.io/jupyter/scipy-notebook:latest AS build
# fix: https://github.com/hadolint/hadolint/wiki/DL4006
# fix: https://github.com/koalaman/shellcheck/wiki/SC3014
SHELL ["/bin/bash", "-o", "pipefail", "-c"]
USER root
# general purpose utils
RUN apt-get update --yes && \
apt-get install --yes --no-install-recommends \
git-lfs \
htop \
iputils-ping && \
apt-get clean && rm -rf /var/lib/apt/lists/*
# install code-server and extensions
ENV CODE_VERSION=4.100.1
RUN wget --no-hsts -q https://github.com/coder/code-server/releases/download/v$CODE_VERSION/code-server_${CODE_VERSION}_amd64.deb && \
dpkg -i code-server_${CODE_VERSION}_amd64.deb && \
rm -f code-server_${CODE_VERSION}_amd64.deb && \
code-server --force \
--install-extension ms-python.python \
--install-extension ms-toolsai.jupyter \
--install-extension eamodio.gitlens \
--install-extension gitlab.gitlab-workflow && \
mkdir -p /usr/local/bin/start-notebook.d && \
chown -R ${NB_USER} "/home/${NB_USER}/.config" "/home/${NB_USER}/.local" && \
fix-permissions "/home/${NB_USER}"
# install VS Code CLI
RUN wget --no-hsts -q -O vscode_cli.tar.gz 'https://code.visualstudio.com/sha/download?build=stable&os=cli-alpine-x64' && \
tar -xf vscode_cli.tar.gz --directory /usr/local/bin && \
rm vscode_cli.tar.gz && \
chown ${NB_USER} /usr/local/bin/code && \
fix-permissions /usr/local/bin/code && \
# fix issue with permissions of /opt/conda/share/gdb/auto-load/opt
fix-permissions "${CONDA_DIR}"
# install some packages
USER ${NB_UID}
RUN mamba install --yes \
# system monitor
'bpytop' \
# umap + hdbscan
'umap-learn' \
'hdbscan' \
# gradio
'gradio' \
# optuna + plotly
'optuna' \
'plotly' \
# optional dependencies for pandas for read_html and to support Parquet files
'lxml' \
'pyarrow' \
'pyogrio' \
# geopandas
'geopandas' \
'pysal' \
# networkx + pyvis + netgraph
'networkx' \
'pyvis' \
'netgraph' \
# dependencies for jupyter-vscode-proxy (?)
'rfc3339-validator' \
'rfc3986-validator' \
'uri-template' \ 'fqdn' \
'webcolors' \
'isoduration' \
'jsonpointer' \
# provide a way for lecturers to share code
'nbgitpuller' \
# monitor GPUs in terminal
'nvtop' \
# integration of VS Code in JupyterLab
'jupyter-server-proxy' \
'jupyter-vscode-proxy' \
# some improvements of jupyterlab
'jupyterlab_execute_time' \
'jupyter-archive' \
'jupyter-resource-usage' \
# install opencv headless version
'py-opencv=*=headless*' && \
pip install --no-cache-dir --extra-index-url 'https://pypi.nvidia.com' --extra-index-url 'https://download.pytorch.org/whl/cu118' \
'transformers' \
'diffusers' \
'datasets' \
'timm' \
'torch' \
'torchaudio' \
'torchvision' && \
printf '%s\n' \
"" \
"# track CPU usage, prevent known bug with prometheus, set default limits if not set by MEM_LIMIT and CPU_LIMIT" \
"import os" \
"" \
"c.ResourceUseDisplay.track_cpu_percent = True" \
"c.ResourceUseDisplay.enable_prometheus_metrics = False" \
"c.ResourceUseDisplay.mem_limit = int(os.getenv('MEM_LIMIT', os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')))" \
"c.ResourceUseDisplay.cpu_limit = float(os.getenv('CPU_LIMIT', os.cpu_count()))" \
>> /etc/jupyter/jupyter_notebook_config.py && \
mamba clean --all -f -y && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
# default settings and extensions in /etc/skel
USER root
# activate extensions by default
COPY plugin.jupyterlab-settings "/home/${NB_USER}/.jupyter/lab/user-settings/@jupyterlab/extensionmanager-extension/"
# source /etc/profile.d/mamba-conda.sh to init mamba and conda (workaround for removed mamba init --system, see https://github.com/mamba-org/mamba/issues/3691)
COPY mamba-conda.sh "/etc/profile.d/"
RUN chown -R ${NB_USER} "/home/${NB_USER}/.jupyter" && \
# history search with Page Up/Down
sed -i "s/^# \(.*history-search.*\)/\1/" /etc/inputrc && \
# VS Code Python settings
mkdir -p "/home/${NB_USER}/.local/share/code-server/User" && \
echo -e '{\n "python.defaultInterpreterPath": "/opt/conda/bin/python",\n "jupyter.sendSelectionToInteractiveWindow": true\n}' > /home/${NB_USER}/.local/share/code-server/User/settings.json && \
mkdir -p "/home/${NB_USER}/.local/share/code-server/Machine" && \
cp "/home/${NB_USER}/.local/share/code-server/User/settings.json" "/home/${NB_USER}/.local/share/code-server/Machine/settings.json" && \
mkdir -p /etc/skel/.local/share/ && \
cp -r "/home/${NB_USER}/.local/share/code-server" /etc/skel/.local/share/ && \
# matplotlib cache
mkdir -p /etc/skel/.cache && \
cp -r "/home/${NB_USER}/.cache/matplotlib" /etc/skel/.cache/ && \
# stuff
mkdir -p "/home/${NB_USER}/.conda" && \
cp -r "/home/${NB_USER}/.conda" "/home/${NB_USER}/.config" "/home/${NB_USER}/.jupyter" /etc/skel/ && \
# VS Code extension "gitlab-workflow", default gitlab server: gitlab.cvh-server.de
printf '%s\n' \
'# Set default gitlab server to gitlab.cvh-server.de' \
'relpath=".local/share/code-server/extensions/gitlab.gitlab-workflow-*/extension.js"' \
'for abspath in /home/${NB_USER}/${relpath} /etc/skel/${relpath}; do' \
' if [[ -e "${abspath}" ]]; then' \
' sed -i "s_=\"https://gitlab.com\"_=\"https://gitlab.cvh-server.de\"_g" "${abspath}"' \
' fi' \
'done' \ >> /usr/local/bin/start-notebook.d/fix-gitlab-server.sh && \
# activate mamba also in graphical terminals
cat /etc/profile.d/mamba-conda.sh >> /etc/bash.bashrc && \
# fix permissions
chown -R ${NB_USER} "/home/${NB_USER}/.local" && \
fix-permissions "/home/${NB_USER}" && \
fix-permissions "/etc/skel"
# remember to copy back in post start hook
# override maintainer label from Jupyter docker stacks
LABEL maintainer="Christof Kaufmann <christof.kaufmann@hs-bochum.de>"
# OCI annotations, see https://github.com/opencontainers/image-spec/blob/main/annotations.md
ARG LABEL_CREATED
ARG LABEL_REVISION=test-build
LABEL org.opencontainers.image.created=$LABEL_CREATED
LABEL org.opencontainers.image.authors="Christof Kaufmann <christof.kaufmann@hs-bochum.de>"
LABEL org.opencontainers.image.source="https://gitlab.cvh-server.de/ckaufmann/gpu-cluster-images"
LABEL org.opencontainers.image.revision=$LABEL_REVISION
LABEL org.opencontainers.image.vendor="UAS Bochum"
LABEL org.opencontainers.image.licenses=BSD-3-Clause
LABEL org.opencontainers.image.title="Jupyter Notebook PyTorch GPU image"
LABEL org.opencontainers.image.description="This image includes PyTorch with GPU support, Huggingface, VS Code CLI, code-server and nbgitpuller."
LABEL org.opencontainers.image.base.name=quay.io/jupyter/scipy-notebook:latest
# switch back to jovyan to avoid accidental container runs as root
USER ${NB_UID}
WORKDIR "${HOME}"
###################################################################
######################## Testing the image ########################
###################################################################
FROM build AS test
# replace home directory with skel, which is closer to the kubernetes environment
USER root
RUN cd / && \
rm -r "/home/${NB_USER}" && \
cp -r /etc/skel "/home/${NB_USER}" && \
fix-permissions "/home/${NB_USER}"
USER ${NB_UID}
# enable logging for all Jupyter applications
RUN printf '%s\n' \
"" \
"# Filter out user connection hint message, which is a CRITICAL logger message" \
"import logging" \
"class UserHintFilter(logging.Filter):" \
" def filter(self, record):" \
" return 'To access the server' not in record.getMessage()" \
"" \
"c.Application.logging_config = {" \
" 'filters': {" \
" 'user_hint': {" \
" '()': UserHintFilter," \
" }," \
" }," \
" 'formatters': {" \
" 'file': {" \
" 'format': '%(asctime)s %(levelname)-8s %(name)-15s %(message)s'," \
" }," \
" }," \
" 'handlers': {" \
" 'file': {" \
" 'class': 'logging.FileHandler'," \
" 'filters': ['user_hint']," \
" 'formatter': 'file'," \
" 'level': 'INFO'," \
" 'filename': '/home/jovyan/jupyter.log'," \ " }," \
" }," \
" 'loggers': {" \
" 'Application': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
" 'loggers': {" \
" 'JupyterApp': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
" 'loggers': {" \
" 'ExtensionApp': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
" 'loggers': {" \
" 'LabServerApp': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
" 'loggers': {" \
" 'LabApp': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
" 'loggers': {" \
" 'NotebookApp': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
" 'loggers': {" \
" 'ServerApp': {" \
" 'level': 'DEBUG'," \
" 'handlers': ['console', 'file']," \
" }," \
" }," \
"}" \
>> /etc/jupyter/jupyter_notebook_config.py
# collect lab logs for 4 sec
RUN start-notebook.sh & sleep 4 && kill -INT %1 && sleep 1
# check log file:
# not existing means some error prevents logging → bad
RUN [[ -f jupyter.log ]] \
|| ( echo "Log file jupyter.log does not exist!" && false ) \
# existing, contains error → bad, print log file
&& ! grep -qP '(Traceback|ERROR|CRITICAL)' jupyter.log \
|| ( cat jupyter.log && false )
# add some environment variables and collect lab logs again for 4 sec
RUN mv jupyter.log jupyter1.log
ENV MEM_LIMIT=1000000000 \
CPU_LIMIT=16.0
RUN start-notebook.sh & sleep 4 && kill -INT %1 && sleep 1
# check log file:
# not existing means some error prevents logging → bad
RUN [[ -f jupyter.log ]] \
|| ( echo "Log file jupyter.log does not exist!" && false ) \ # existing, contains error → bad, print log file
&& ! grep -qP '(Traceback|ERROR|CRITICAL)' jupyter.log \
|| ( cat jupyter.log && false )