This commit is contained in:
Cornelius Specht 2024-02-23 12:29:52 +01:00
parent e6879297e7
commit 7f60a5db31
7 changed files with 319 additions and 6 deletions

View File

@ -9,9 +9,7 @@ steps:
registry: git.sandbox.iuk.hdm-stuttgart.de registry: git.sandbox.iuk.hdm-stuttgart.de
repo: specht/woodpecker-test repo: specht/woodpecker-test
cache: false cache: false
# tag: 0.0.1 tag: git.sandbox.iuk.hdm-stuttgart.de/specht/woodpecker-test:test
context: Dockerfile context: Dockerfile
token: token:
from_secret: secret_token from_secret: secret_token
when:
branch: audio

View File

@ -1 +1,183 @@
FROM hello-world:linux FROM python:3.10.8-slim-bullseye
# debian:bullseye-20230109-slim
ARG NB_USER="jovyan"
ARG NB_UID="1000"
ARG NB_GID="100"
RUN apt-get update --yes && \
# - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as
# the ubuntu base image is rebuilt too seldom sometimes (less than once a month)
apt-get upgrade --yes && \
apt-get install --yes --no-install-recommends \
# - bzip2 is necessary to extract the micromamba executable.
bzip2 \
ca-certificates \
ffmpeg \
git \
git-lfs \
locales \
curl \
libportaudio2 \
dnsutils \
sudo \
nano \
build-essential \
tini \
wget && \
apt-get clean && rm -rf /var/lib/apt/lists/* && \
echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \
locale-gen
# Configure environment
ENV CONDA_DIR=/opt/conda \
SHELL=/bin/bash \
NB_USER="${NB_USER}" \
NB_UID=${NB_UID} \
NB_GID=${NB_GID} \
LC_ALL=en_US.UTF-8 \
LANG=en_US.UTF-8 \
LANGUAGE=en_US.UTF-8
ENV PATH="${CONDA_DIR}/bin:${PATH}" \
HOME="/home/${NB_USER}"
# Copy a script that we will use to correct permissions after running certain commands
COPY fix-permissions /usr/local/bin/fix-permissions
RUN chmod a+rx /usr/local/bin/fix-permissions
# Enable prompt color in the skeleton .bashrc before creating the default NB_USER
# hadolint ignore=SC2016
RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \
# Add call to conda init script see https://stackoverflow.com/a/58081608/4413446
echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc
# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group
# and make sure these dirs are writable by the `users` group.
RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \
sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \
sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \
useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \
mkdir -p "${CONDA_DIR}" && \
chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \
chmod g+w /etc/passwd && \
fix-permissions "${HOME}" && \
fix-permissions "${CONDA_DIR}"
USER ${NB_UID}
# Pin python version here, or set it to "default"
ARG PYTHON_VERSION=3.10
# Setup work directory for backward-compatibility
RUN mkdir "/home/${NB_USER}/work" && \
fix-permissions "/home/${NB_USER}"
COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc"
WORKDIR /tmp
RUN set -x && \
arch=$(uname -m) && \
if [ "${arch}" = "x86_64" ]; then \
# Should be simpler, see <https://github.com/mamba-org/mamba/issues/1437>
arch="64"; \
fi && \
wget -qO /tmp/micromamba.tar.bz2 \
"https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \
tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \
rm /tmp/micromamba.tar.bz2 && \
PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \
if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \
# Install the packages
./micromamba install \
--root-prefix="${CONDA_DIR}" \
--prefix="${CONDA_DIR}" \
--yes \
"${PYTHON_SPECIFIER}" \
'mamba' \
'jupyter_core' && \
rm micromamba && \
# Pin major.minor version of python
mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \
mamba clean --all -f -y && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
#RUN conda update --all --yes
#RUN conda install -c conda-forge tensorflow-gpu=2.11 -y
#RUN CONDA_CUDA_OVERRIDE="11.2" mamba install --yes -c conda-forge tensorflow==2.10.0=cuda112py310*
ENV CONDA_CUDA_OVERRIDE="11.2"
ENV NUM_GPUS=1
#RUN mamba install -y tensorflow cudatoolkit>=11.2 -c conda-forge
#==2.11.0=cuda*
#RUN mamba install -y -c conda-forge tensorflow-gpu
RUN CONDA_OVERRIDE_CUDA="11.2" conda install tensorflow-gpu==2.11.0 -c conda-forge
RUN CONDA_OVERRIDE_CUDA="11.2" conda install -c conda-forge pytorch-gpu
WORKDIR /tmp
RUN mamba install --quiet --yes \
'notebook==6.4.12' \
'jupyterhub==2.3.1' \
'mathjax' \
'voila==0.3.7' \
'jupyterlab-git' \
'numba' \
'cython' \
'matplotlib' \
'pandas' \
'scipy' \
'scikit-image' \
'scikit-learn' \
'jupyterlab==3.4.3' \
'ipywebrtc' && \
jupyter notebook --generate-config && \
mamba clean --all -f -y && \
# npm cache clean --force && \
jupyter lab clean && \
rm -rf "/home/${NB_USER}/.cache/yarn" && \
fix-permissions "${CONDA_DIR}" && \
fix-permissions "/home/${NB_USER}"
ENV JUPYTER_PORT=8888
EXPOSE $JUPYTER_PORT
#RUN pip install ipywidgets voila jupyter-collaboration mathjax
RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-sandbox-theme
#RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-data-pool
#RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-training
#RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-synthetic-data
RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-sandbox-announcement
RUN pip install sounddevice wavio
RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116
RUN pip install opencv-python-headless
# Configure container startup
CMD ["start-notebook.sh"]
# Copy local files as late as possible to avoid cache busting
COPY start-notebook.sh start-singleuser.sh /usr/local/bin/
# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab
COPY jupyter_server_config.py /etc/jupyter/
# Fix permissions on /etc/jupyter as root
USER root
RUN echo 'export PATH=/opt/conda/bin:$PATH' >> /etc/profile
#RUN source /etc/profile
# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205)
RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \
/etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \
fix-permissions /etc/jupyter/
# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck
# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands
# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799
HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \
CMD wget -O- --no-verbose --tries=1 --no-check-certificate \
http${GEN_CERT:+s}://localhost:${JUPYTER_PORT}${JUPYTERHUB_SERVICE_PREFIX:-/}api || exit 1
# Switch back to jovyan to avoid accidental container runs as root
USER ${NB_UID}
RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements"
#USER root
WORKDIR "${HOME}"

35
fix-permissions Normal file
View File

@ -0,0 +1,35 @@
#!/bin/bash
# set permissions on a directory
# after any installation, if a directory needs to be (human) user-writable,
# run this script on it.
# It will make everything in the directory owned by the group ${NB_GID}
# and writable by that group.
# Deployments that want to set a specific user id can preserve permissions
# by adding the `--group-add users` line to `docker run`.
# uses find to avoid touching files that already have the right permissions,
# which would cause massive image explosion
# right permissions are:
# group=${NB_GID}
# AND permissions include group rwX (directory-execute)
# AND directories have setuid,setgid bits set
set -e
for d in "$@"; do
find "${d}" \
! \( \
-group "${NB_GID}" \
-a -perm -g+rwX \
\) \
-exec chgrp "${NB_GID}" {} \; \
-exec chmod g+rwX {} \;
# setuid, setgid *on directories only*
find "${d}" \
\( \
-type d \
-a ! -perm -6000 \
\) \
-exec chmod +6000 {} \;
done

6
initial-condarc Normal file
View File

@ -0,0 +1,6 @@
# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html
auto_update_conda: false
show_channel_urls: true
channels:
- conda-forge

57
jupyter_server_config.py Normal file
View File

@ -0,0 +1,57 @@
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
# mypy: ignore-errors
import os
import stat
import subprocess
from jupyter_core.paths import jupyter_data_dir
c = get_config() # noqa: F821
c.ServerApp.ip = "0.0.0.0"
c.ServerApp.port = 8888
c.ServerApp.open_browser = False
# https://github.com/jupyter/notebook/issues/3130
c.FileContentsManager.delete_to_trash = False
# Generate a self-signed certificate
OPENSSL_CONFIG = """\
[req]
distinguished_name = req_distinguished_name
[req_distinguished_name]
"""
if "GEN_CERT" in os.environ:
dir_name = jupyter_data_dir()
pem_file = os.path.join(dir_name, "notebook.pem")
os.makedirs(dir_name, exist_ok=True)
# Generate an openssl.cnf file to set the distinguished name
cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf")
if not os.path.isfile(cnf_file):
with open(cnf_file, "w") as fh:
fh.write(OPENSSL_CONFIG)
# Generate a certificate if one doesn't exist on disk
subprocess.check_call(
[
"openssl",
"req",
"-new",
"-newkey=rsa:2048",
"-days=365",
"-nodes",
"-x509",
"-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated",
f"-keyout={pem_file}",
f"-out={pem_file}",
]
)
# Restrict access to the file
os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR)
c.ServerApp.certfile = pem_file
# Change default umask for all subprocesses of the notebook server if set in
# the environment
if "NB_UMASK" in os.environ:
os.umask(int(os.environ["NB_UMASK"], 8))

22
start-notebook.sh Normal file
View File

@ -0,0 +1,22 @@
#!/bin/bash
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
set -e
# The Jupyter command to launch
# JupyterLab by default
DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}"
if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then
echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub."
exec /usr/local/bin/start-singleuser.sh "$@"
fi
wrapper=""
if [[ "${RESTARTABLE}" == "yes" ]]; then
wrapper="run-one-constantly"
fi
# shellcheck disable=SC1091,SC2086
exec /usr/local/bin/start.sh ${wrapper} jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@"

13
start-singleuser.sh Normal file
View File

@ -0,0 +1,13 @@
#!/bin/bash
# Copyright (c) Jupyter Development Team.
# Distributed under the terms of the Modified BSD License.
set -e
# set default ip to 0.0.0.0
if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then
NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}"
fi
# shellcheck disable=SC1091,SC2086
. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@"