diff --git a/.woodpecker.yml b/.woodpecker.yml index 1d67a7a..9710116 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -1,16 +1,16 @@ steps: - publish-container: - image: taywee/woodpecker-buildah:amd64 - pull: true + build_and_release: + image: maltegrosse/woodpecker-buildah:0.0.12 environment: - CONDA_CUDA_OVERRIDE=11.2 - NVIDIA_VISIBLE_DEVICES=all settings: registry: git.sandbox.iuk.hdm-stuttgart.de - repo: grosse/jupyterlab-datascience-gpu - cache: false - tag: git.sandbox.iuk.hdm-stuttgart.de/grosse/jupyterlab-datascience-gpu:test + repository: grosse/jupyterlab-datascience-gpu + tag: 4.0.x-gpu-torch + architectures: aarch64 amd64 context: Dockerfile + imagename: jupyterlab-datascience username: from_secret: docker_username password: diff --git a/Dockerfile b/Dockerfile index d68ab9e..e376062 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,183 +1,5 @@ -FROM python:3.10.8-slim-bullseye +FROM git.sandbox.iuk.hdm-stuttgart.de/grosse/jupyterlab-datascience:4.0.12c # debian:bullseye-20230109-slim ARG NB_USER="jovyan" ARG NB_UID="1000" -ARG NB_GID="100" - -RUN apt-get update --yes && \ - # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as - # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) - apt-get upgrade --yes && \ - apt-get install --yes --no-install-recommends \ - # - bzip2 is necessary to extract the micromamba executable. - bzip2 \ - ca-certificates \ - ffmpeg \ - git \ - git-lfs \ - locales \ - curl \ - libportaudio2 \ - dnsutils \ - sudo \ - nano \ - build-essential \ - tini \ - wget && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen - - -# Configure environment -ENV CONDA_DIR=/opt/conda \ - SHELL=/bin/bash \ - NB_USER="${NB_USER}" \ - NB_UID=${NB_UID} \ - NB_GID=${NB_GID} \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 -ENV PATH="${CONDA_DIR}/bin:${PATH}" \ - HOME="/home/${NB_USER}" - -# Copy a script that we will use to correct permissions after running certain commands -COPY fix-permissions /usr/local/bin/fix-permissions -RUN chmod a+rx /usr/local/bin/fix-permissions - -# Enable prompt color in the skeleton .bashrc before creating the default NB_USER -# hadolint ignore=SC2016 -RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ - # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc - -# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group -# and make sure these dirs are writable by the `users` group. -RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ - sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ - sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ - mkdir -p "${CONDA_DIR}" && \ - chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ - chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" - -USER ${NB_UID} - -# Pin python version here, or set it to "default" -ARG PYTHON_VERSION=3.10 - -# Setup work directory for backward-compatibility -RUN mkdir "/home/${NB_USER}/work" && \ - fix-permissions "/home/${NB_USER}" - -COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" -WORKDIR /tmp -RUN set -x && \ - arch=$(uname -m) && \ - if [ "${arch}" = "x86_64" ]; then \ - # Should be simpler, see - arch="64"; \ - fi && \ - wget -qO /tmp/micromamba.tar.bz2 \ - "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ - tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ - rm /tmp/micromamba.tar.bz2 && \ - PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ - if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - # Install the packages - ./micromamba install \ - --root-prefix="${CONDA_DIR}" \ - --prefix="${CONDA_DIR}" \ - --yes \ - "${PYTHON_SPECIFIER}" \ - 'mamba' \ - 'jupyter_core' && \ - rm micromamba && \ - # Pin major.minor version of python - mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -#RUN conda update --all --yes -#RUN conda install -c conda-forge tensorflow-gpu=2.11 -y - -#RUN CONDA_CUDA_OVERRIDE="11.2" mamba install --yes -c conda-forge tensorflow==2.10.0=cuda112py310* -ENV CONDA_CUDA_OVERRIDE="11.2" -ENV NUM_GPUS=1 -#RUN mamba install -y tensorflow cudatoolkit>=11.2 -c conda-forge -#==2.11.0=cuda* -#RUN mamba install -y -c conda-forge tensorflow-gpu -RUN CONDA_OVERRIDE_CUDA="11.2" conda install tensorflow-gpu==2.11.0 -c conda-forge -RUN CONDA_OVERRIDE_CUDA="11.2" conda install -c conda-forge pytorch-gpu -WORKDIR /tmp -RUN mamba install --quiet --yes \ - 'notebook==6.4.12' \ - 'jupyterhub==2.3.1' \ - 'mathjax' \ - 'voila==0.3.7' \ - 'jupyterlab-git' \ - 'numba' \ - 'cython' \ - 'matplotlib' \ - 'pandas' \ - 'scipy' \ - 'scikit-image' \ - 'scikit-learn' \ - 'jupyterlab==3.4.3' \ - 'ipywebrtc' && \ - jupyter notebook --generate-config && \ - mamba clean --all -f -y && \ - # npm cache clean --force && \ - jupyter lab clean && \ - rm -rf "/home/${NB_USER}/.cache/yarn" && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -ENV JUPYTER_PORT=8888 -EXPOSE $JUPYTER_PORT - - - -#RUN pip install ipywidgets voila jupyter-collaboration mathjax -RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-sandbox-theme -#RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-data-pool -#RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-training -#RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-synthetic-data -RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-sandbox-announcement -RUN pip install sounddevice wavio -RUN pip install torch==1.13.1+cu116 torchvision==0.14.1+cu116 torchaudio==0.13.1 --extra-index-url https://download.pytorch.org/whl/cu116 -RUN pip install opencv-python-headless -# Configure container startup -CMD ["start-notebook.sh"] - -# Copy local files as late as possible to avoid cache busting -COPY start-notebook.sh start-singleuser.sh /usr/local/bin/ -# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab -COPY jupyter_server_config.py /etc/jupyter/ - -# Fix permissions on /etc/jupyter as root -USER root - -RUN echo 'export PATH=/opt/conda/bin:$PATH' >> /etc/profile -#RUN source /etc/profile - -# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) -RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ - /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ - fix-permissions /etc/jupyter/ - -# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck -# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands -# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 -HEALTHCHECK --interval=5s --timeout=3s --start-period=5s --retries=3 \ - CMD wget -O- --no-verbose --tries=1 --no-check-certificate \ - http${GEN_CERT:+s}://localhost:${JUPYTER_PORT}${JUPYTERHUB_SERVICE_PREFIX:-/}api || exit 1 - -# Switch back to jovyan to avoid accidental container runs as root -USER ${NB_UID} -RUN jupyter labextension disable "@jupyterlab/apputils-extension:announcements" -#USER root -WORKDIR "${HOME}" \ No newline at end of file +ARG NB_GID="100" \ No newline at end of file diff --git a/fix-permissions b/fix-permissions deleted file mode 100644 index 0969275..0000000 --- a/fix-permissions +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# set permissions on a directory -# after any installation, if a directory needs to be (human) user-writable, -# run this script on it. -# It will make everything in the directory owned by the group ${NB_GID} -# and writable by that group. -# Deployments that want to set a specific user id can preserve permissions -# by adding the `--group-add users` line to `docker run`. - -# uses find to avoid touching files that already have the right permissions, -# which would cause massive image explosion - -# right permissions are: -# group=${NB_GID} -# AND permissions include group rwX (directory-execute) -# AND directories have setuid,setgid bits set - -set -e - -for d in "$@"; do - find "${d}" \ - ! \( \ - -group "${NB_GID}" \ - -a -perm -g+rwX \ - \) \ - -exec chgrp "${NB_GID}" {} \; \ - -exec chmod g+rwX {} \; - # setuid, setgid *on directories only* - find "${d}" \ - \( \ - -type d \ - -a ! -perm -6000 \ - \) \ - -exec chmod +6000 {} \; -done \ No newline at end of file diff --git a/initial-condarc b/initial-condarc deleted file mode 100644 index 66ecf60..0000000 --- a/initial-condarc +++ /dev/null @@ -1,6 +0,0 @@ -# Conda configuration see https://conda.io/projects/conda/en/latest/configuration.html - -auto_update_conda: false -show_channel_urls: true -channels: - - conda-forge \ No newline at end of file diff --git a/jupyter_server_config.py b/jupyter_server_config.py deleted file mode 100644 index 160c4b5..0000000 --- a/jupyter_server_config.py +++ /dev/null @@ -1,57 +0,0 @@ -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -# mypy: ignore-errors -import os -import stat -import subprocess - -from jupyter_core.paths import jupyter_data_dir - -c = get_config() # noqa: F821 -c.ServerApp.ip = "0.0.0.0" -c.ServerApp.port = 8888 -c.ServerApp.open_browser = False - -# https://github.com/jupyter/notebook/issues/3130 -c.FileContentsManager.delete_to_trash = False - -# Generate a self-signed certificate -OPENSSL_CONFIG = """\ -[req] -distinguished_name = req_distinguished_name -[req_distinguished_name] -""" -if "GEN_CERT" in os.environ: - dir_name = jupyter_data_dir() - pem_file = os.path.join(dir_name, "notebook.pem") - os.makedirs(dir_name, exist_ok=True) - - # Generate an openssl.cnf file to set the distinguished name - cnf_file = os.path.join(os.getenv("CONDA_DIR", "/usr/lib"), "ssl", "openssl.cnf") - if not os.path.isfile(cnf_file): - with open(cnf_file, "w") as fh: - fh.write(OPENSSL_CONFIG) - - # Generate a certificate if one doesn't exist on disk - subprocess.check_call( - [ - "openssl", - "req", - "-new", - "-newkey=rsa:2048", - "-days=365", - "-nodes", - "-x509", - "-subj=/C=XX/ST=XX/L=XX/O=generated/CN=generated", - f"-keyout={pem_file}", - f"-out={pem_file}", - ] - ) - # Restrict access to the file - os.chmod(pem_file, stat.S_IRUSR | stat.S_IWUSR) - c.ServerApp.certfile = pem_file - -# Change default umask for all subprocesses of the notebook server if set in -# the environment -if "NB_UMASK" in os.environ: - os.umask(int(os.environ["NB_UMASK"], 8)) \ No newline at end of file diff --git a/old.woodpecker.yml b/old.woodpecker.yml deleted file mode 100644 index a89c0ba..0000000 --- a/old.woodpecker.yml +++ /dev/null @@ -1,13 +0,0 @@ -pipeline: - publish-container: - image: woodpeckerci/plugin-docker-buildx:2.1.0 - secrets: [ docker_username, docker_password ] - group: docker - environment: - - CONDA_CUDA_OVERRIDE=11.2 - - NVIDIA_VISIBLE_DEVICES=all - settings: - registry: https://git.sandbox.iuk.hdm-stuttgart.de - repo: git.sandbox.iuk.hdm-stuttgart.de/grosse/jupyterlab-datascience-gpu - dockerfile: Dockerfile - tags: latest diff --git a/old/Dockerfile b/old/Dockerfile deleted file mode 100644 index 6f08cac..0000000 --- a/old/Dockerfile +++ /dev/null @@ -1,393 +0,0 @@ -ARG NVIDIA_IMAGE=nvcr.io/nvidia/tensorflow:22.12-tf2-py3 - -FROM ${NVIDIA_IMAGE} -############################################################################ -#################### Dependency: jupyter/base-image ######################## -############################################################################ - -#### copied from https://github.com/jupyter/docker-stacks/tree/main/docker-stacks-foundation - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -# Ubuntu 22.04 (jammy) -# https://hub.docker.com/_/ubuntu/tags?page=1&name=jammy -ARG ROOT_CONTAINER=ubuntu:22.04 - - -LABEL maintainer="Jupyter Project " -ARG NB_USER="jovyan" -ARG NB_UID="1000" -ARG NB_GID="100" - -# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 -# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -USER root - -# Install all OS dependencies for notebook server that starts but lacks all -# features (e.g., download as all possible file formats) -ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update --yes && \ - # - apt-get upgrade is run to patch known vulnerabilities in apt-get packages as - # the ubuntu base image is rebuilt too seldom sometimes (less than once a month) - apt-get upgrade --yes && \ - apt-get install --yes --no-install-recommends \ - # - bzip2 is necessary to extract the micromamba executable. - bzip2 \ - ca-certificates \ - locales \ - sudo \ - # - tini is installed as a helpful container entrypoint that reaps zombie - # processes and such of the actual executable we want to start, see - # https://github.com/krallin/tini#why-tini for details. - tini \ - wget && \ - apt-get clean && rm -rf /var/lib/apt/lists/* && \ - echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && \ - locale-gen - -# Configure environment -ENV CONDA_DIR=/opt/conda \ - SHELL=/bin/bash \ - NB_USER="${NB_USER}" \ - NB_UID=${NB_UID} \ - NB_GID=${NB_GID} \ - LC_ALL=en_US.UTF-8 \ - LANG=en_US.UTF-8 \ - LANGUAGE=en_US.UTF-8 -ENV PATH="${CONDA_DIR}/bin:${PATH}" \ - HOME="/home/${NB_USER}" - -# Copy a script that we will use to correct permissions after running certain commands -COPY fix-permissions /usr/local/bin/fix-permissions -RUN chmod a+rx /usr/local/bin/fix-permissions - -# Enable prompt color in the skeleton .bashrc before creating the default NB_USER -# hadolint ignore=SC2016 -RUN sed -i 's/^#force_color_prompt=yes/force_color_prompt=yes/' /etc/skel/.bashrc && \ - # Add call to conda init script see https://stackoverflow.com/a/58081608/4413446 - echo 'eval "$(command conda shell.bash hook 2> /dev/null)"' >> /etc/skel/.bashrc - -# Create NB_USER with name jovyan user with UID=1000 and in the 'users' group -# and make sure these dirs are writable by the `users` group. -RUN echo "auth requisite pam_deny.so" >> /etc/pam.d/su && \ - sed -i.bak -e 's/^%admin/#%admin/' /etc/sudoers && \ - sed -i.bak -e 's/^%sudo/#%sudo/' /etc/sudoers && \ - useradd -l -m -s /bin/bash -N -u "${NB_UID}" "${NB_USER}" && \ - mkdir -p "${CONDA_DIR}" && \ - chown "${NB_USER}:${NB_GID}" "${CONDA_DIR}" && \ - chmod g+w /etc/passwd && \ - fix-permissions "${HOME}" && \ - fix-permissions "${CONDA_DIR}" - -USER ${NB_UID} - -# Pin python version here, or set it to "default" -ARG PYTHON_VERSION=3.10 - -# Setup work directory for backward-compatibility -RUN mkdir "/home/${NB_USER}/work" && \ - fix-permissions "/home/${NB_USER}" - -# Download and install Micromamba, and initialize Conda prefix. -# -# Similar projects using Micromamba: -# - Micromamba-Docker: -# - repo2docker: -# Install Python, Mamba and jupyter_core -# Cleanup temporary files and remove Micromamba -# Correct permissions -# Do all this in a single RUN command to avoid duplicating all of the -# files across image layers when the permissions change - -COPY --chown="${NB_UID}:${NB_GID}" initial-condarc "${CONDA_DIR}/.condarc" -WORKDIR /tmp -RUN set -x && \ - arch=$(uname -m) && \ - if [ "${arch}" = "x86_64" ]; then \ - # Should be simpler, see - arch="64"; \ - fi && \ - wget -qO /tmp/micromamba.tar.bz2 \ - "https://micromamba.snakepit.net/api/micromamba/linux-${arch}/latest" && \ - tar -xvjf /tmp/micromamba.tar.bz2 --strip-components=1 bin/micromamba && \ - rm /tmp/micromamba.tar.bz2 && \ - PYTHON_SPECIFIER="python=${PYTHON_VERSION}" && \ - if [[ "${PYTHON_VERSION}" == "default" ]]; then PYTHON_SPECIFIER="python"; fi && \ - # Install the packages - ./micromamba install \ - --root-prefix="${CONDA_DIR}" \ - --prefix="${CONDA_DIR}" \ - --yes \ - "${PYTHON_SPECIFIER}" \ - 'mamba' \ - 'jupyter_core' && \ - rm micromamba && \ - # Pin major.minor version of python - mamba list python | grep '^python ' | tr -s ' ' | cut -d ' ' -f 1,2 >> "${CONDA_DIR}/conda-meta/pinned" && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -EXPOSE 8888 - -# Configure container startup -ENTRYPOINT ["tini", "-g", "--"] -CMD ["start-notebook.sh"] - -# Copy local files as late as possible to avoid cache busting -COPY start.sh start-notebook.sh start-singleuser.sh /usr/local/bin/ -# Currently need to have both jupyter_notebook_config and jupyter_server_config to support classic and lab -COPY jupyter_server_config.py /etc/jupyter/ - -# Fix permissions on /etc/jupyter as root -USER root - -# Legacy for Jupyter Notebook Server, see: [#1205](https://github.com/jupyter/docker-stacks/issues/1205) -RUN sed -re "s/c.ServerApp/c.NotebookApp/g" \ - /etc/jupyter/jupyter_server_config.py > /etc/jupyter/jupyter_notebook_config.py && \ - fix-permissions /etc/jupyter/ - -# HEALTHCHECK documentation: https://docs.docker.com/engine/reference/builder/#healthcheck -# This healtcheck works well for `lab`, `notebook`, `nbclassic`, `server` and `retro` jupyter commands -# https://github.com/jupyter/docker-stacks/issues/915#issuecomment-1068528799 -HEALTHCHECK --interval=15s --timeout=3s --start-period=5s --retries=3 \ - CMD wget -O- --no-verbose --tries=1 --no-check-certificate \ - http${GEN_CERT:+s}://localhost:8888${JUPYTERHUB_SERVICE_PREFIX:-/}api || exit 1 - -# Switch back to jovyan to avoid accidental container runs as root -USER ${NB_UID} - -WORKDIR "${HOME}" - - -############################################################################ -################# Dependency: jupyter/minimal-notebook ##################### -############################################################################ - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter - -LABEL maintainer="Jupyter Project " - -# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 -# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -USER root - -# Install all OS dependencies for fully functional notebook server -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - # Common useful utilities - git \ - nano-tiny \ - tzdata \ - unzip \ - vim-tiny \ - # Inkscape is installed to be able to convert SVG files - inkscape \ - # git-over-ssh - openssh-client \ - # less is needed to run help in R - # see: https://github.com/jupyter/docker-stacks/issues/1588 - less \ - # nbconvert dependencies - # https://nbconvert.readthedocs.io/en/latest/install.html#installing-tex - texlive-xetex \ - texlive-fonts-recommended \ - texlive-plain-generic && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -# Create alternative for nano -> nano-tiny -RUN update-alternatives --install /usr/bin/nano nano /bin/nano-tiny 10 - -# Switch back to jovyan to avoid accidental container runs as root -USER ${NB_UID} - -############################################################################ -################# Dependency: jupyter/scipy-notebook ####################### -############################################################################ - -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. -ARG OWNER=jupyter - -LABEL maintainer="Jupyter Project " - -# Fix: https://github.com/hadolint/hadolint/wiki/DL4006 -# Fix: https://github.com/koalaman/shellcheck/wiki/SC3014 -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -USER root - -RUN apt-get update --yes && \ - apt-get install --yes --no-install-recommends \ - # for cython: https://cython.readthedocs.io/en/latest/src/quickstart/install.html - build-essential \ - # for latex labels - cm-super \ - dvipng \ - # for matplotlib anim - ffmpeg && \ - apt-get clean && rm -rf /var/lib/apt/lists/* - -USER ${NB_UID} - -# Install Python 3 packages -RUN mamba install --quiet --yes \ - 'altair' \ - 'beautifulsoup4' \ - 'bokeh' \ - 'bottleneck' \ - 'cloudpickle' \ - 'conda-forge::blas=*=openblas' \ - 'cython' \ - 'dask' \ - 'dill' \ - 'h5py' \ - 'ipympl'\ - 'ipywidgets' \ - 'matplotlib-base' \ - 'numba' \ - 'numexpr' \ - 'pandas' \ - 'patsy' \ - 'protobuf' \ - 'pytables' \ - 'scikit-image' \ - 'scikit-learn' \ - 'scipy' \ - 'seaborn' \ - 'sqlalchemy' \ - 'statsmodels' \ - 'sympy' \ - 'widgetsnbextension'\ - 'xlrd' && \ - mamba clean --all -f -y && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Install facets which does not have a pip or conda package at the moment -WORKDIR /tmp -RUN git clone https://github.com/PAIR-code/facets.git && \ - jupyter nbextension install facets/facets-dist/ --sys-prefix && \ - rm -rf /tmp/facets && \ - fix-permissions "${CONDA_DIR}" && \ - fix-permissions "/home/${NB_USER}" - -# Import matplotlib the first time to build the font cache. -ENV XDG_CACHE_HOME="/home/${NB_USER}/.cache/" - -RUN MPLBACKEND=Agg python -c "import matplotlib.pyplot" && \ - fix-permissions "/home/${NB_USER}" - -USER ${NB_UID} - -WORKDIR "${HOME}" - - - -################ copied from https://github.com/iot-salzburg/gpu-jupyter/tree/master/src - - -# LABEL authors="Christoph Schranz , Mathematical Michael " - - #tensorflow 2.10.1 requires keras<2.11,>=2.10.0, but you have keras 2.11.0 - -# Install Tensorflow, check compatibility here: -# https://www.tensorflow.org/install/source#gpu -# installation via conda leads to errors in version 4.8.2 -RUN pip install --upgrade pip && \ - pip install --no-cache-dir "tensorflow==2.11.0" -RUN pip install --upgrade pip && \ - pip install --no-cache-dir keras==2.11.0 - -# Install PyTorch with dependencies -RUN conda install --quiet --yes \ - pyyaml mkl mkl-include setuptools cmake cffi typing && \ - conda clean --all -f -y && \ - fix-permissions $CONDA_DIR && \ - fix-permissions /home/$NB_USER - -# Check compatibility here: -# https://pytorch.org/get-started/locally/ -# Installation via conda leads to errors installing cudatoolkit=11.1 -RUN pip install --no-cache-dir torch torchvision torchaudio torchviz --extra-index-url https://download.pytorch.org/whl/cu116 - -ENV CUDA_PATH=/opt/conda/ - -USER root - -# Install nvtop to monitor the gpu tasks -RUN apt-get update && \ - apt-get install -y cmake libncurses5-dev libncursesw5-dev git && \ - rm -rf /var/lib/apt/lists/* - -RUN git clone https://github.com/Syllo/nvtop.git /run/nvtop && \ - mkdir -p /run/nvtop/build && cd /run/nvtop/build && \ - (cmake .. -DNVML_RETRIEVE_HEADER_ONLINE=True 2> /dev/null || echo "cmake was not successful") && \ - (make 2> /dev/null || echo "make was not successful") && \ - (make install 2> /dev/null || echo "make install was not successful") && \ - cd /tmp && rm -rf /tmp/nvtop - -RUN fix-permissions /home/$NB_USER - -USER $NB_UID - - - ############################################################################ - ############################ Useful packages ############################### - ############################################################################ - -LABEL authors="Christoph Schranz , Mathematical Michael " - -USER root - -RUN pip install --no-cache-dir ipyleaflet "plotly>=4.14.3" "ipywidgets>=7.5" -RUN conda install -c conda-forge nodejs -# Install important packages and Graphviz -RUN set -ex \ - && buildDeps=' \ - graphviz==0.19.1 \ -' \ - && apt-get update \ - && apt-get -y install htop apt-utils iputils-ping graphviz libgraphviz-dev openssh-client \ - && pip install --no-cache-dir $buildDeps - -# Install various extensions -RUN fix-permissions $CONDA_DIR - -# RUN jupyter labextension install @jupyterlab/github -# RUN pip install jupyterlab-git -RUN pip install jupyterlab-drawio -RUN jupyter nbextension enable --py --sys-prefix ipyleaflet - -RUN jupyter labextension install jupyterlab-plotly -RUN jupyter labextension install @jupyter-widgets/jupyterlab-manager plotlywidget -# RUN pip install --no-cache-dir jupyter-tabnine --user && \ -# jupyter nbextension install --py jupyter_tabnine --user && \ -# jupyter nbextension enable --py jupyter_tabnine --user && \ -# jupyter serverextension enable --py jupyter_tabnine --user -RUN pip install --no-cache-dir jupyter_contrib_nbextensions \ - jupyter_nbextensions_configurator rise -# jupyter nbextension enable codefolding/main -RUN jupyter labextension install @ijmbarr/jupyterlab_spellchecker - -RUN fix-permissions /home/$NB_USER - - -## add sandbox plugins -RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-sandbox-theme -RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-data-pool -RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-training -RUN pip install --extra-index-url https://git.sandbox.iuk.hdm-stuttgart.de/api/packages/grosse/pypi/simple jupyterlab-synthetic-data - -# Switch back to jovyan to avoid accidental container runs as root -USER $NB_UID - -# Copy jupyter_notebook_config.json -#COPY jupyter_notebook_config.json /etc/jupyter/ \ No newline at end of file diff --git a/old/jupyter_notebook_config.json b/old/jupyter_notebook_config.json deleted file mode 100644 index 2669c3b..0000000 --- a/old/jupyter_notebook_config.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "NotebookApp": { - "password": "sha1:3b4b6378355:642693b20f0a33bcad27b94293d0ed7db3408322" - } - } \ No newline at end of file diff --git a/old/start.sh b/old/start.sh deleted file mode 100644 index fef6b61..0000000 --- a/old/start.sh +++ /dev/null @@ -1,262 +0,0 @@ -#!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -set -e - -# The _log function is used for everything this script wants to log. It will -# always log errors and warnings, but can be silenced for other messages -# by setting JUPYTER_DOCKER_STACKS_QUIET environment variable. -_log () { - if [[ "$*" == "ERROR:"* ]] || [[ "$*" == "WARNING:"* ]] || [[ "${JUPYTER_DOCKER_STACKS_QUIET}" == "" ]]; then - echo "$@" - fi -} -_log "Entered start.sh with args:" "$@" - -# The run-hooks function looks for .sh scripts to source and executable files to -# run within a passed directory. -run-hooks () { - if [[ ! -d "${1}" ]] ; then - return - fi - _log "${0}: running hooks in ${1} as uid / gid: $(id -u) / $(id -g)" - for f in "${1}/"*; do - case "${f}" in - *.sh) - _log "${0}: running script ${f}" - # shellcheck disable=SC1090 - source "${f}" - ;; - *) - if [[ -x "${f}" ]] ; then - _log "${0}: running executable ${f}" - "${f}" - else - _log "${0}: ignoring non-executable ${f}" - fi - ;; - esac - done - _log "${0}: done running hooks in ${1}" -} - -# A helper function to unset env vars listed in the value of the env var -# JUPYTER_ENV_VARS_TO_UNSET. -unset_explicit_env_vars () { - if [ -n "${JUPYTER_ENV_VARS_TO_UNSET}" ]; then - for env_var_to_unset in $(echo "${JUPYTER_ENV_VARS_TO_UNSET}" | tr ',' ' '); do - echo "Unset ${env_var_to_unset} due to JUPYTER_ENV_VARS_TO_UNSET" - unset "${env_var_to_unset}" - done - unset JUPYTER_ENV_VARS_TO_UNSET - fi -} - - -# Default to starting bash if no command was specified -if [ $# -eq 0 ]; then - cmd=( "bash" ) -else - cmd=( "$@" ) -fi - -# NOTE: This hook will run as the user the container was started with! -run-hooks /usr/local/bin/start-notebook.d - -# If the container started as the root user, then we have permission to refit -# the jovyan user, and ensure file permissions, grant sudo rights, and such -# things before we run the command passed to start.sh as the desired user -# (NB_USER). -# -if [ "$(id -u)" == 0 ] ; then - # Environment variables: - # - NB_USER: the desired username and associated home folder - # - NB_UID: the desired user id - # - NB_GID: a group id we want our user to belong to - # - NB_GROUP: a group name we want for the group - # - GRANT_SUDO: a boolean ("1" or "yes") to grant the user sudo rights - # - CHOWN_HOME: a boolean ("1" or "yes") to chown the user's home folder - # - CHOWN_EXTRA: a comma separated list of paths to chown - # - CHOWN_HOME_OPTS / CHOWN_EXTRA_OPTS: arguments to the chown commands - - # Refit the jovyan user to the desired the user (NB_USER) - if id jovyan &> /dev/null ; then - if ! usermod --home "/home/${NB_USER}" --login "${NB_USER}" jovyan 2>&1 | grep "no changes" > /dev/null; then - _log "Updated the jovyan user:" - _log "- username: jovyan -> ${NB_USER}" - _log "- home dir: /home/jovyan -> /home/${NB_USER}" - fi - elif ! id -u "${NB_USER}" &> /dev/null; then - _log "ERROR: Neither the jovyan user or '${NB_USER}' exists. This could be the result of stopping and starting, the container with a different NB_USER environment variable." - exit 1 - fi - # Ensure the desired user (NB_USER) gets its desired user id (NB_UID) and is - # a member of the desired group (NB_GROUP, NB_GID) - if [ "${NB_UID}" != "$(id -u "${NB_USER}")" ] || [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then - _log "Update ${NB_USER}'s UID:GID to ${NB_UID}:${NB_GID}" - # Ensure the desired group's existence - if [ "${NB_GID}" != "$(id -g "${NB_USER}")" ]; then - groupadd --force --gid "${NB_GID}" --non-unique "${NB_GROUP:-${NB_USER}}" - fi - # Recreate the desired user as we want it - userdel "${NB_USER}" - useradd --home "/home/${NB_USER}" --uid "${NB_UID}" --gid "${NB_GID}" --groups 100 --no-log-init "${NB_USER}" - fi - - # Move or symlink the jovyan home directory to the desired users home - # directory if it doesn't already exist, and update the current working - # directory to the new location if needed. - if [[ "${NB_USER}" != "jovyan" ]]; then - if [[ ! -e "/home/${NB_USER}" ]]; then - _log "Attempting to copy /home/jovyan to /home/${NB_USER}..." - mkdir "/home/${NB_USER}" - if cp -a /home/jovyan/. "/home/${NB_USER}/"; then - _log "Success!" - else - _log "Failed to copy data from /home/jovyan to /home/${NB_USER}!" - _log "Attempting to symlink /home/jovyan to /home/${NB_USER}..." - if ln -s /home/jovyan "/home/${NB_USER}"; then - _log "Success creating symlink!" - else - _log "ERROR: Failed copy data from /home/jovyan to /home/${NB_USER} or to create symlink!" - exit 1 - fi - fi - fi - # Ensure the current working directory is updated to the new path - if [[ "${PWD}/" == "/home/jovyan/"* ]]; then - new_wd="/home/${NB_USER}/${PWD:13}" - _log "Changing working directory to ${new_wd}" - cd "${new_wd}" - fi - fi - - # Optionally ensure the desired user get filesystem ownership of it's home - # folder and/or additional folders - if [[ "${CHOWN_HOME}" == "1" || "${CHOWN_HOME}" == "yes" ]]; then - _log "Ensuring /home/${NB_USER} is owned by ${NB_UID}:${NB_GID} ${CHOWN_HOME_OPTS:+(chown options: ${CHOWN_HOME_OPTS})}" - # shellcheck disable=SC2086 - chown ${CHOWN_HOME_OPTS} "${NB_UID}:${NB_GID}" "/home/${NB_USER}" - fi - if [ -n "${CHOWN_EXTRA}" ]; then - for extra_dir in $(echo "${CHOWN_EXTRA}" | tr ',' ' '); do - _log "Ensuring ${extra_dir} is owned by ${NB_UID}:${NB_GID} ${CHOWN_EXTRA_OPTS:+(chown options: ${CHOWN_EXTRA_OPTS})}" - # shellcheck disable=SC2086 - chown ${CHOWN_EXTRA_OPTS} "${NB_UID}:${NB_GID}" "${extra_dir}" - done - fi - - # Update potentially outdated environment variables since image build - export XDG_CACHE_HOME="/home/${NB_USER}/.cache" - - # Prepend ${CONDA_DIR}/bin to sudo secure_path - sed -r "s#Defaults\s+secure_path\s*=\s*\"?([^\"]+)\"?#Defaults secure_path=\"${CONDA_DIR}/bin:\1\"#" /etc/sudoers | grep secure_path > /etc/sudoers.d/path - - # Optionally grant passwordless sudo rights for the desired user - if [[ "$GRANT_SUDO" == "1" || "$GRANT_SUDO" == "yes" ]]; then - _log "Granting ${NB_USER} passwordless sudo rights!" - echo "${NB_USER} ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers.d/added-by-start-script - fi - - # NOTE: This hook is run as the root user! - run-hooks /usr/local/bin/before-notebook.d - - unset_explicit_env_vars - _log "Running as ${NB_USER}:" "${cmd[@]}" - exec sudo --preserve-env --set-home --user "${NB_USER}" \ - PATH="${PATH}" \ - PYTHONPATH="${PYTHONPATH:-}" \ - "${cmd[@]}" - # Notes on how we ensure that the environment that this container is started - # with is preserved (except vars listed in JUPYTER_ENV_VARS_TO_UNSET) when - # we transition from running as root to running as NB_USER. - # - # - We use `sudo` to execute the command as NB_USER. What then - # happens to the environment will be determined by configuration in - # /etc/sudoers and /etc/sudoers.d/* as well as flags we pass to the sudo - # command. The behavior can be inspected with `sudo -V` run as root. - # - # ref: `man sudo` https://linux.die.net/man/8/sudo - # ref: `man sudoers` https://www.sudo.ws/man/1.8.15/sudoers.man.html - # - # - We use the `--preserve-env` flag to pass through most environment - # variables, but understand that exceptions are caused by the sudoers - # configuration: `env_delete` and `env_check`. - # - # - We use the `--set-home` flag to set the HOME variable appropriately. - # - # - To reduce the default list of variables deleted by sudo, we could have - # used `env_delete` from /etc/sudoers. It has higher priority than the - # `--preserve-env` flag and the `env_keep` configuration. - # - # - We preserve PATH and PYTHONPATH explicitly. Note however that sudo - # resolves `${cmd[@]}` using the "secure_path" variable we modified - # above in /etc/sudoers.d/path. Thus PATH is irrelevant to how the above - # sudo command resolves the path of `${cmd[@]}`. The PATH will be relevant - # for resolving paths of any subprocesses spawned by `${cmd[@]}`. - -# The container didn't start as the root user, so we will have to act as the -# user we started as. -else - # Warn about misconfiguration of: granting sudo rights - if [[ "${GRANT_SUDO}" == "1" || "${GRANT_SUDO}" == "yes" ]]; then - _log "WARNING: container must be started as root to grant sudo permissions!" - fi - - JOVYAN_UID="$(id -u jovyan 2>/dev/null)" # The default UID for the jovyan user - JOVYAN_GID="$(id -g jovyan 2>/dev/null)" # The default GID for the jovyan user - - # Attempt to ensure the user uid we currently run as has a named entry in - # the /etc/passwd file, as it avoids software crashing on hard assumptions - # on such entry. Writing to the /etc/passwd was allowed for the root group - # from the Dockerfile during build. - # - # ref: https://github.com/jupyter/docker-stacks/issues/552 - if ! whoami &> /dev/null; then - _log "There is no entry in /etc/passwd for our UID=$(id -u). Attempting to fix..." - if [[ -w /etc/passwd ]]; then - _log "Renaming old jovyan user to nayvoj ($(id -u jovyan):$(id -g jovyan))" - - # We cannot use "sed --in-place" since sed tries to create a temp file in - # /etc/ and we may not have write access. Apply sed on our own temp file: - sed --expression="s/^jovyan:/nayvoj:/" /etc/passwd > /tmp/passwd - echo "${NB_USER}:x:$(id -u):$(id -g):,,,:/home/jovyan:/bin/bash" >> /tmp/passwd - cat /tmp/passwd > /etc/passwd - rm /tmp/passwd - - _log "Added new ${NB_USER} user ($(id -u):$(id -g)). Fixed UID!" - - if [[ "${NB_USER}" != "jovyan" ]]; then - _log "WARNING: user is ${NB_USER} but home is /home/jovyan. You must run as root to rename the home directory!" - fi - else - _log "WARNING: unable to fix missing /etc/passwd entry because we don't have write permission. Try setting gid=0 with \"--user=$(id -u):0\"." - fi - fi - - # Warn about misconfiguration of: desired username, user id, or group id. - # A misconfiguration occurs when the user modifies the default values of - # NB_USER, NB_UID, or NB_GID, but we cannot update those values because we - # are not root. - if [[ "${NB_USER}" != "jovyan" && "${NB_USER}" != "$(id -un)" ]]; then - _log "WARNING: container must be started as root to change the desired user's name with NB_USER=\"${NB_USER}\"!" - fi - if [[ "${NB_UID}" != "${JOVYAN_UID}" && "${NB_UID}" != "$(id -u)" ]]; then - _log "WARNING: container must be started as root to change the desired user's id with NB_UID=\"${NB_UID}\"!" - fi - if [[ "${NB_GID}" != "${JOVYAN_GID}" && "${NB_GID}" != "$(id -g)" ]]; then - _log "WARNING: container must be started as root to change the desired user's group id with NB_GID=\"${NB_GID}\"!" - fi - - # Warn if the user isn't able to write files to ${HOME} - if [[ ! -w /home/jovyan ]]; then - _log "WARNING: no write access to /home/jovyan. Try starting the container with group 'users' (100), e.g. using \"--group-add=users\"." - fi - - # NOTE: This hook is run as the user we started the container as! - run-hooks /usr/local/bin/before-notebook.d - unset_explicit_env_vars - _log "Executing the command:" "${cmd[@]}" - exec "${cmd[@]}" -fi \ No newline at end of file diff --git a/start-notebook.sh b/start-notebook.sh deleted file mode 100644 index cfbdffd..0000000 --- a/start-notebook.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -set -e - -# The Jupyter command to launch -# JupyterLab by default -DOCKER_STACKS_JUPYTER_CMD="${DOCKER_STACKS_JUPYTER_CMD:=lab}" - -if [[ -n "${JUPYTERHUB_API_TOKEN}" ]]; then - echo "WARNING: using start-singleuser.sh instead of start-notebook.sh to start a server associated with JupyterHub." - exec /usr/local/bin/start-singleuser.sh "$@" -fi - -wrapper="" -if [[ "${RESTARTABLE}" == "yes" ]]; then - wrapper="run-one-constantly" -fi - -# shellcheck disable=SC1091,SC2086 -exec /usr/local/bin/start.sh ${wrapper} jupyter ${DOCKER_STACKS_JUPYTER_CMD} ${NOTEBOOK_ARGS} "$@" \ No newline at end of file diff --git a/start-singleuser.sh b/start-singleuser.sh deleted file mode 100644 index 7c49847..0000000 --- a/start-singleuser.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash -# Copyright (c) Jupyter Development Team. -# Distributed under the terms of the Modified BSD License. - -set -e - -# set default ip to 0.0.0.0 -if [[ "${NOTEBOOK_ARGS} $*" != *"--ip="* ]]; then - NOTEBOOK_ARGS="--ip=0.0.0.0 ${NOTEBOOK_ARGS}" -fi - -# shellcheck disable=SC1091,SC2086 -. /usr/local/bin/start.sh jupyterhub-singleuser ${NOTEBOOK_ARGS} "$@" \ No newline at end of file