406 lines
207 KiB
Plaintext
406 lines
207 KiB
Plaintext
|
{
|
||
|
|
"cells": [
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "8413e039",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"*this notebook requires a working Tensorflow GPU environment* "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "1434db93-27e6-4ed4-bcb5-1fabbdf155d5",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"# Tensorflow model for Image recognition\n",
|
||
|
|
"Inside the following Use Case their is a CNN trained on the CIFAR-10 Dataset to classify different images. \n",
|
||
|
|
"\n",
|
||
|
|
"*For long running trainings this Notebook can be converted to plain python (using nbconvert). An example repository can be found [here](https://git.sandbox.iuk.hdm-stuttgart.de/grosse/test-ci).*"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "b5ec6c62-d6a7-4b13-b802-c7de5a3716a4",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## Import Libaries"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 1,
|
||
|
|
"id": "e1b70492-eba5-421b-8b51-d56632657efc",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"2024-05-27 08:17:22.992136: E external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:9261] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
|
||
|
|
"2024-05-27 08:17:22.992198: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:607] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
|
||
|
|
"2024-05-27 08:17:22.993032: E external/local_xla/xla/stream_executor/cuda/cuda_blas.cc:1515] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
|
||
|
|
"2024-05-27 08:17:22.998566: I tensorflow/core/platform/cpu_feature_guard.cc:182] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
|
||
|
|
"To enable the following instructions: SSE4.1 SSE4.2 AVX AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"import tensorflow as tf\n",
|
||
|
|
"from tensorflow import keras\n",
|
||
|
|
"import requests\n",
|
||
|
|
"import numpy as np\n",
|
||
|
|
"import os\n",
|
||
|
|
"# Version Information\n",
|
||
|
|
"# tensorflow 2.2.0 , Cudnn7.6.5 and Cuda 10.1 , python 3.8\n",
|
||
|
|
"from keras import backend as K\n",
|
||
|
|
"K.clear_session()\n",
|
||
|
|
"import matplotlib.pyplot as plt "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "085824f7-0423-4ede-8203-76b0d673bb04",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## Select available Hardware for model execution"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 2,
|
||
|
|
"id": "2067ff7e-f5fc-4ad2-a95d-29d82e3f7059",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"2024-05-27 08:17:25.521816: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:25.554614: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:25.556557: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"gpus = tf.config.experimental.list_physical_devices('GPU')\n",
|
||
|
|
"if gpus:\n",
|
||
|
|
" try:\n",
|
||
|
|
" tf.config.experimental.set_virtual_device_configuration(gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=6024)])\n",
|
||
|
|
" except RuntimeError as e:\n",
|
||
|
|
" print(e)\n",
|
||
|
|
" os.exit(1)\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "fbff826d-33ac-42f7-8a18-f3df0ec01b5f",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## Show some dataset details "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 3,
|
||
|
|
"id": "d9d12acc-d312-4b32-87dc-131148012c4d",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"[PhysicalDevice(name='/physical_device:CPU:0', device_type='CPU'), PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')]\n",
|
||
|
|
"2.15.0\n",
|
||
|
|
"True\n",
|
||
|
|
"(50000, 32, 32, 3) (50000, 1)\n",
|
||
|
|
"deer\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"print(tf.config.experimental.list_physical_devices())\n",
|
||
|
|
"\n",
|
||
|
|
"print(tf.__version__)\n",
|
||
|
|
"\n",
|
||
|
|
"print(tf.test.is_built_with_cuda())\n",
|
||
|
|
"\n",
|
||
|
|
"(X_train, y_train), (X_test,y_test) = tf.keras.datasets.cifar10.load_data()\n",
|
||
|
|
"\n",
|
||
|
|
"(X_train, y_train), (X_test,y_test) = tf.keras.datasets.cifar10.load_data()\n",
|
||
|
|
"\n",
|
||
|
|
"print(X_train.shape,y_train.shape)\n",
|
||
|
|
"\n",
|
||
|
|
"classes = [\"airplane\",\"automobile\",\"bird\",\"cat\",\"deer\",\"dog\",\"frog\",\"horse\",\"ship\",\"truck\"]\n",
|
||
|
|
"\n",
|
||
|
|
"print(classes[y_train[3][0]])\n",
|
||
|
|
"\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "59ac7149-9708-463a-a214-8e01ee71b04d",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## image scaling "
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 4,
|
||
|
|
"id": "fb66a6f8-b4f9-4e85-9251-87c6ae403632",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"X_train_scaled = X_train / 255\n",
|
||
|
|
"X_test_scaled = X_test / 255\n",
|
||
|
|
"\n",
|
||
|
|
"y_train_categorical = keras.utils.to_categorical(\n",
|
||
|
|
" y_train, num_classes=10, dtype='float32'\n",
|
||
|
|
")\n",
|
||
|
|
"y_test_categorical = keras.utils.to_categorical(\n",
|
||
|
|
" y_test, num_classes=10, dtype='float32'\n",
|
||
|
|
")"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "28bb8a0e-52af-4cf4-999e-008e464398aa",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## show dataset with categories"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 5,
|
||
|
|
"id": "e14ec124-9d54-43a3-9ea1-4b63383590dd",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAxoAAAMpCAYAAACDrkVRAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAAEAAElEQVR4nOz9d5xk6VnfDV8nVa7u6pymJ4edzUm72l0lFBDIQhIyBmOwENYrwDYiCCEe/CLAGD5YYIzxY14eg3mECLZwkiykRSBW2tXmMJt3Z3Zy7O7pWF25Tnz/6J7u/v3u2p0ZUdOjcH312Y/m6qo6933uc4dzqn6/+7KSJElEURRFURRFURSli9hXuwKKoiiKoiiKonzroQ8aiqIoiqIoiqJ0HX3QUBRFURRFURSl6+iDhqIoiqIoiqIoXUcfNBRFURRFURRF6Tr6oKEoiqIoiqIoStfRBw1FURRFURRFUbqOeylviuNYpqampFgsimVZV7pOyjcBSZJItVqV8fFxse0r+7yq/U9hNrP/iWgfVBDtf8rVRtdg5WpyOf3vkh40pqamZHJysiuVU761OHPmjGzZsuWKlqH9T3k1NqP/iWgfVDqj/U+52ugarFxNLqX/XdKDRrFYFBGR2+64U1x35SPLy0vwnrQdQ9yXMhOOb+nLQTzYj/FAbx7ilO1B7KSzeEDHMcpYKi9DHIRYj1JvL8R2FEDc9tsQt1oYZ7Jpo8xIIoibzTrEPb1F/ECC7/d9rIPT4bI4dK6FfAHifA7b0vUyELfaPlbB6vAEamO5vo+fCZP1bzJabV8+8R//Yq1vXEkulPHJP/6vklk9z6nDz8J75k+9AnEUmW04vGUvxFt27IO4NIKDJZPFYxw9+DjEp4+/aJQR1vDaO1SPYqkHYjeN1+22198N8c7dWOdWBcediMjBl5+HOI7xugVhC+JDB1+GuLq8ADGPARGRMMD+t7TYhLjWwDLCCOswONgHcakPx7qISJzU8Bghvt5qro/lIAjly3/ztU3pfyLrffDMmTPS07NyDeM4fq2PfPNCUzd/g9msN4yPLC5hH+rrK0EcBdinslmcy50Uzqud5qdYsB7m7L95VCoV2bZt26b3v9GhjNj2SjtksjjH83VyLbOF+JvHMMa1SOgYy5UqxBk7BXHONsuotXEusHN4bTMpXNfzeZwLenpwjS6Xcc7zG+b8xHcbAa2p1HXEcbHeKRfbpSePbSsiMjpYgnhqdhbiho9tWSzi+0O6F2nUK0YZ4+PYnzwP1w/XWY+DMJIv3ndwU9fgP/iTz0k2t3K9eP7LprBveBmzDRMH+8LGewoREZdGtU3d0+MpNzHvMxO6loFlvmcjVkSvJ9g/owBfj7hSIkb/MupE9eSYPx/HZp0jehO/g48Zcxx1qDcfg+LQqPf6BWg26vLzH3rbJfW/S3rQuDCBua679qDBN76OTR3GMRsq5eFn0jSIeAJKORi7aYzFMavfpGPYNtYjQ8fgPmMJ9WSaiLmOIiIRWV1iurnkMiXB99t0eR0xy+D2ztIxsxka5B7G/GvnpTxoOPQZnhRWjnvlf0a9UEYml1ub5NI0iaVokuv0oMGfydLDWY4e3vhBI0M3R+m0+dBp80Mj1YM/42YwztGiW6BB7Ma0gIpILof1imPsK36A1yidxrZqU59OeAyIiEULgOsGFFN7WzhueMFMdRpHCb6Hu1YUmnPKZv2Mf6Gcnp6eb/sHDa/DvBuE+GB5oY0uEPl488lj75vtQeMCm93/bNtae9Bw6KGB68Kvr3we/5bwTRgdw7ZfO+5cxkU+49ivGbv0EGC8v0OZPDPE/B5+0LAvUgfHLMOjevF7HLrn4fPgm2Ius1MZRpl8TNncNTiby0vu1R40aG1LdXjQiI0HDWp3GtXOFXjQ4E/YF3nQCL9ZHzTo+kRdftC4wKX0PzWDK4qiKIqiKIrSdS7pF40LHDp0UKzVbwLK8/PwWj89vFoD5tPsYITfzlrZYYjr8SLENXrSTCz8JrbRwm/RREQaTfxZNYjwCWyevqbPuFhGGOL7Hfu1v5FeqQfKZUKSrlitAYj51+agTbIC12y7GkmfFiPUlVz4lmGtTJKdWfTrkHT4VqjRwm+pw4C+nXfXz70dkK5lE6iWl9baaqDUD68lQyMYu/iNqojI2NadEEf064AdoyQkbuA5tkgekjTxW1oRkYlB7NNbJ3dDPLl7G8TjEyjXGh7G8/A8+gaohN8Ei4hMbhnF99C3y60WypzKSyhRmp/HceemzP4nJMPoGyA5RB7LWCaJVzqD4yhOzP7juXjMynIZYr+9PlbDq9D/mM0wAX8j0m4sG39bPHsc4jMH8T3LFZwj73nr2yDuyXKfM9vWom/0rmbrX61r7znO2i8EUYjzV0xrnUW/8oqItEmPyBIi/kWjVMT5pod+cfWreF1FROImzj85D39x7aVfYHN07Qv0a+c8relxYkqnMvTL8NDQIMRLSzgfsexsfAznbcf4bldkeBjXHI+OceLMFMQpj9qyRL9Wm+pRGSBpN/f5emNDe0eb/4tqbK38J2IqTHxSf9SXUXYnIuLl6dct6htCqgn+FTOkXyeilvkLf2sZ16IU9Y2IfrGvNXE9tC18fyGP16TTL/4sS+Jv+S/26wP/sNjpFw1uC/5RhH/B4DL4F41Ov0TEVFNDfrWhjEv5heQC354rpaIoiqIoiqIoVxR90FAURVEURVEUpevog4aiKIqiKIqiKF3nsjwaGXd9xwshq8I28mRsH0Fdm4jI8BBqHLPsK+BtFGmbvBZtkZh00JilaGcgoZ1qkhiP0Utb7PIOAynSEHaSpfGOKW3aYSUIsZ45er+bxzIyKdMHElqohbXJ/R/yjizUNIU8nmetwxaVAWl+abMQqVbWddd+cOn6vK4RBCKrux35baxro4G64O17J4yP1+rYhn6A16l/EPus6+Fz+J49uNXs3a+/3ShjgrbI7e0dgjhwsd1ypB91eTMK0lQ366gnFRFpk5cml8Vr3VdC/fGunddCfPAgbg0slql7bbexv/T24Ha1tMmZLFfOQ5wIXp9OGtSlJbw+TdrGcqNcNIyuvkfD2DnkWwQ+L5sExDNnThifef7Rr0EcNLG/eAXsL80Kejh6+nFtYD2yiLkT1dVs/at17T3XXtvh0aL26BtEL2C9ac7xXoSejJDmF4vOa2wU547RISzjxNFjRhmDLs6jo+PoIbND2nWR1nH26wzQ9vCJQ2u8iPSStyFH651j43kOjaCHg3eTrFZMH1KY4LzYW8IyJ+hegzdncz18Pe2Y63xMW+T2FNFrmATr674vm78GV+u1tZ2IAlp35ufQw3j2HG7/KyLiZHhXRZwX0jbvPoef99mX1MGr16jiGpkln6NQKoaqj14S38dCd+7YA/HuXeizFBHJ0g5b7JcwdijkXUDpD3GnLXl5c6yL7WR1ETp5NGyuRwc/yteD/qKhKIqiKIqiKErX0QcNRVEURVEURVG6jj5oKIqiKIqiKIrSdS7Po2FFYlsrmq1iET+6dwK1dgNZM4OlF6MmvraIuu0oxueeJuUxsEkH3lPCTM4iIi75G8q0lzMnMO6nfcKrtN+7Tzkymh32bWZ9XYH2Gg983NfZpmzRHuXmiCKzDJdMF23yKKRIJG/H2HbtGu4jLpwNU0TSdMlC0hUu19c18364+Xt4h62WhKu6QitEfWo6hbrdZcrzIiIyMIr+ia3XYY6L4clxiDm7upA+NAjNPBqHplGn2jg+h5+xsc+/8sJzEL9uP/on3nTH6yDupMOskJ749Cnez52zqKPud3AI/SynzxwxykhlyOPTxHFRqWB7u7SHfE8Pfr7ZQT/OtgvOaQMZzb8B7BGblRV6s+F94gPy50ydOWV8podzI5RQVz+7hPPwwvQ5iEcmt+IBOdmQmJfcYhPZJnK1rn1vsbCWUZpzQQwPo59idgHnIhGRDK01y0tliEcG0VOWpkUhm0Uvw8Qk+i9ERPLG+ocDOyU4r6ZpzW40cb2cHMfzSoz00CKpNB7T93GeHRwg/x3p9NttnM+KPWa+omYb61VdxjW13cY1aWAQx0A2j+u+a5keC9fH82jVscx
|
||
|
|
"text/plain": [
|
||
|
|
"<Figure size 1000x1000 with 25 Axes>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"plt.figure(figsize=(10,10))\n",
|
||
|
|
"for i in range(25):\n",
|
||
|
|
" plt.subplot(5,5,i+1)\n",
|
||
|
|
" plt.xticks([])\n",
|
||
|
|
" plt.yticks([])\n",
|
||
|
|
" plt.grid(False)\n",
|
||
|
|
" plt.imshow(X_train_scaled[i])\n",
|
||
|
|
" # The CIFAR labels happen to be arrays, \n",
|
||
|
|
" # which is why you need the extra index\n",
|
||
|
|
" plt.xlabel(classes[y_train[i][0]])\n",
|
||
|
|
"plt.show()"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "7a2c7eff-b9a7-4ded-ac0a-2d117f5cf532",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## configure model"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 6,
|
||
|
|
"id": "196ab4a0-45a5-479a-8b0c-86b8e6f2daa0",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"2024-05-27 08:17:26.939176: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:26.942340: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:26.944200: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:27.074726: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:27.076028: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:27.077228: I external/local_xla/xla/stream_executor/cuda/cuda_executor.cc:901] successful NUMA node read from SysFS had negative value (-1), but there must be at least one NUMA node, so returning NUMA node zero. See more at https://github.com/torvalds/linux/blob/v6.0/Documentation/ABI/testing/sysfs-bus-pci#L344-L355\n",
|
||
|
|
"2024-05-27 08:17:27.078400: I tensorflow/core/common_runtime/gpu/gpu_device.cc:1929] Created device /job:localhost/replica:0/task:0/device:GPU:0 with 6024 MB memory: -> device: 0, name: NVIDIA A100 80GB PCIe MIG 3g.40gb, pci bus id: 0000:05:00.0, compute capability: 8.0\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"Epoch 1/5\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stderr",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"2024-05-27 08:17:29.263602: I external/local_xla/xla/service/service.cc:168] XLA service 0x7fcd9f189fa0 initialized for platform CUDA (this does not guarantee that XLA will be used). Devices:\n",
|
||
|
|
"2024-05-27 08:17:29.263822: I external/local_xla/xla/service/service.cc:176] StreamExecutor device (0): NVIDIA A100 80GB PCIe MIG 3g.40gb, Compute Capability 8.0\n",
|
||
|
|
"2024-05-27 08:17:29.278180: I external/local_xla/xla/stream_executor/cuda/cuda_dnn.cc:454] Loaded cuDNN version 8907\n",
|
||
|
|
"WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
|
||
|
|
"I0000 00:00:1716797849.319990 330 device_compiler.h:186] Compiled cluster using XLA! This line is logged at most once for the lifetime of the process.\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"1563/1563 [==============================] - 7s 4ms/step - loss: 1.8087 - accuracy: 0.3562\n",
|
||
|
|
"Epoch 2/5\n",
|
||
|
|
"1563/1563 [==============================] - 7s 4ms/step - loss: 1.6230 - accuracy: 0.4242\n",
|
||
|
|
"Epoch 3/5\n",
|
||
|
|
"1563/1563 [==============================] - 7s 4ms/step - loss: 1.5412 - accuracy: 0.4558\n",
|
||
|
|
"Epoch 4/5\n",
|
||
|
|
"1563/1563 [==============================] - 6s 4ms/step - loss: 1.4829 - accuracy: 0.4778\n",
|
||
|
|
"Epoch 5/5\n",
|
||
|
|
"1563/1563 [==============================] - 7s 4ms/step - loss: 1.4331 - accuracy: 0.4957\n",
|
||
|
|
"finished training\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"\n",
|
||
|
|
"def get_model():\n",
|
||
|
|
" model = keras.Sequential([\n",
|
||
|
|
" keras.layers.Flatten(input_shape=(32,32,3)),\n",
|
||
|
|
" keras.layers.Dense(3000, activation='relu'),\n",
|
||
|
|
" keras.layers.Dense(1000, activation='relu'),\n",
|
||
|
|
" keras.layers.Dense(10, activation='sigmoid') \n",
|
||
|
|
" ])\n",
|
||
|
|
"\n",
|
||
|
|
" model.compile(optimizer='SGD',\n",
|
||
|
|
" loss='categorical_crossentropy',\n",
|
||
|
|
" metrics=['accuracy'])\n",
|
||
|
|
" return model\n",
|
||
|
|
"history = None\n",
|
||
|
|
"with tf.device('/GPU:0'):\n",
|
||
|
|
" model = keras.Sequential([\n",
|
||
|
|
" keras.layers.Flatten(input_shape=(32,32,3)),\n",
|
||
|
|
" keras.layers.Dense(3000, activation='relu'),\n",
|
||
|
|
" keras.layers.Dense(1000, activation='relu'),\n",
|
||
|
|
" keras.layers.Dense(10, activation='sigmoid') \n",
|
||
|
|
" ])\n",
|
||
|
|
" model.compile(optimizer='SGD',\n",
|
||
|
|
" loss='categorical_crossentropy',\n",
|
||
|
|
" metrics=['accuracy'])\n",
|
||
|
|
" history = model.fit(X_train_scaled, y_train_categorical, epochs=5)\n",
|
||
|
|
" model.save('mymodel.keras')\n",
|
||
|
|
" \n",
|
||
|
|
" print(\"finished training\")"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "98c4e3c1-39ab-48fc-97c6-97bd0349b396",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## show result"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 7,
|
||
|
|
"id": "970f7cd9-8b78-423b-8067-aaf666c31cd9",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"data": {
|
||
|
|
"image/png": "iVBORw0KGgoAAAANSUhEUgAAAiwAAAGdCAYAAAAxCSikAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8g+/7EAAAACXBIWXMAAA9hAAAPYQGoP6dpAABTu0lEQVR4nO3dd1yW9f7H8dcNyFABB85w4FZwby1zpVmadhrWKVPLPJZaHk/LtmXRMq1TWnZMj2lpv0itHDnKPVFRc+dIQ9C0BEEBhev3x/cwbkEFBa77hvfz8bgep2vdfAiPvPtOh2VZFiIiIiIuzMPuAkRERESuRoFFREREXJ4Ci4iIiLg8BRYRERFxeQosIiIi4vIUWERERMTlKbCIiIiIy1NgEREREZfnZXcB+SUtLY3jx4/j7++Pw+GwuxwRERHJBcuyOHv2LFWrVsXD4/LtKEUmsBw/fpxq1arZXYaIiIhcg2PHjhEcHHzZ+0UmsPj7+wPmGw4ICLC5GhEREcmN+Ph4qlWrlvF7/HKKTGBJ7wYKCAhQYBEREXEzVxvOoUG3IiIi4vIUWERERMTlKbBcRWqqOURERMQ+RWYMS0H54AP45huYNg3q17e7GhER12JZFhcuXODixYt2lyIuysvLixIlSlz3kiP5HlhWrVrFu+++y5YtW4iJiWHu3Ln069fviu/MmjWLd955hwMHDhAYGMitt97Ke++9R/ny5fO7vDw5fx7eeQdOnIBmzeCNN+DJJ8HT09ayRERcQnJyMkeOHCEhIcHuUsTFlS5dmpo1a+Lj43PNn5HvgSUxMZGmTZsyePBg7rrrrqs+v2bNGh566CEmTJhAnz59iI6OZtiwYQwZMoS5c+fmd3l54ucHmzfDkCGwZAn861/w7bemtaVuXVtLExGxVVpaGrt378bLy4uQkBB8fHy0aKdkY1kWycnJREdHs2vXLsLCwvD29r6mz8r3wNKrVy969eqV6+c3bNhAzZo1eeKJJwAICQnhH//4B++8805+l3ZNqlWDxYvhP/8xgWXtWmjaFMLDYeRIuMKifCIiRVZSUhJpaWmEhIRQunRpu8sRF1aqVCm8vb3Zt28fy5Yto2vXrvj6+ub5c2z/dduhQwd+//13Fi5ciGVZnDhxgm+++Ybbb7/9iu8lJycTHx/vdBQUhwMefRR27oRu3UxX0ahR0Lkz/PprgX1ZERGXd6Wl1EXSpf85OXjwIEuWLCH1Gmaz2P4nrUOHDsyaNYv+/fvj7e1N5cqVKVOmDP/+97+v+F54eDiBgYEZR2Esy1+jBixdCpMnQ6lSsHq1aW35978hLa3Av7yIiIhbK1OmDIcPH76mcU+2B5bdu3fzxBNP8PLLL7NlyxYWL17M4cOHGTZs2BXfGzNmDHFxcRnHsWPHCqVehwOGDTOtLV26wLlz8MQT0LUrHDpUKCWIiIgL6dy5M6NGjcr180eOHMHhcBAVFVVgNbmqEiVKcPHiRZKSkvL8ru2BJTw8nI4dO/L000/TpEkTevbsyaRJk/j888+JiYm57Hs+Pj4Zy/DbsRx/SAgsWwYffwwlS8LKldCkCUyapNYWERFX5HA4rngMGjTomj7322+/5fXXX8/189WqVSMmJoawsLBr+nq5VdSCke2B5dy5c9n6QD3/N2/Ysiw7Sso1Dw94/HHT2nLzzZCYCMOHwy23wJEjdlcnIiJZxcTEZBwTJ04kICDA6doHH3zg9PyFCxdy9bnlypW76sZ9WXl6elK5cmW8vLQUWl7ke2BJSEggKioqI9EdPnyYqKgojh49CpiunIceeijj+T59+vDtt98yefJkDh06xNq1a3niiSdo06YNVatWze/yCkStWvDTT/Dhh2Yq9E8/QePG8Mkn4OKZS0Sk2KhcuXLGERgYiMPhyDhPSkqiTJkyfP3113Tu3BlfX19mzpzJ6dOnuf/++wkODqZkyZI0btyYr776yulzL+0SqlmzJm+++SYPP/ww/v7+VK9enSlTpmTcv7TlY8WKFTgcDpYvX06rVq0oWbIkHTp0YN++fU5fZ9y4cVSsWBF/f3+GDBnCc889R7Nmza7530dycjJPPPEEFStWxNfXlxtvvJHNmzdn3P/rr7944IEHqFChAn5+ftStW5dp06YBkJKSwogRI6hSpQq+vr7UrFmT8PDwa64lN/I9sERGRtK8eXOaN28OwOjRo2nevDkvv/wyYBJuengBGDRoEO+//z4fffQRYWFh3HPPPdSvX59vv/02v0srUB4eZprzjh1w442QkACPPQY9esBvv9ldnYhIwbMs09Jc2Ed+/ofhs88+yxNPPMGePXvo2bMnSUlJtGzZkh9++IFffvmFoUOHMmDAADZu3HjFzxk/fjytWrVi27ZtPP744zz22GPs3bv3iu+88MILjB8/nsjISLy8vHj44Ycz7s2aNYs33niDt99+my1btlC9enUmT558Xd/rM888Q0REBP/973/ZunUrderUoWfPnvz5558AvPTSS+zevZtFixaxZ88eJk+eTFBQEAAffvgh3333HV9//TX79u1j5syZ1KxZ87rquSqriIiLi7MAKy4uzu5SrNRUy5owwbL8/CwLLMvf37KmTLGstDS7KxMRyR+JiYlWZGSklZiYmHEtIcH8nVfYR0JC3uufNm2aFRgYmHF++PBhC7AmTpx41Xdvu+0261//+lfG+c0332w9+eSTGec1atSwHnzwwYzztLQ0q2LFitbkyZOdvta2bdssy7Ksn3/+2QKsZcuWZbyzYMECC7DOnz9vWZZltW3b1ho+fLhTHR07drSaNm162Tov/TpZJSQkWCVKlLBmzZqVcS0lJcWqWrWq9c4771iWZVl9+vSxBg8enONnjxw50uratauVlstfbOl/Xr766ivrvffes2JjYzPu5fb3t+1jWIoiDw+zTktUFHToAGfPwtChcOutUEiTmURE5Bq0atXK6Tw1NZU33niDJk2aUL58eUqXLs2SJUucegpy0qRJk4x/Tu96OnnyZK7fqVKlCkDGO/v27aNNmzZOz196nhcHDx7kwoULdOzYMeNaiRIlaNOmDXv27AHgscceY/bs2TRr1oxnnnmGdevWZTw7aNAgoqKiqF+/Pk888QRLliy55lpyS4GlANWrB6tWwfjx4OtrlvcPC4OpUzW2RUSKnpIlTXd4YR8lS+bf91CqVCmn8/HjxzNhwgSeeeYZfvrpJ6KioujZsycpKSlX/JwSJUo4nTscDtKuMoU06zvp2xxkfefSrQ+s6/hFkv5uTp+Zfq1Xr1789ttvjBo1iuPHj9OtWzeeeuopAFq0aMHhw4d5/fXXOX/+PPfeey933333NdeTGwosBczTE0aPNq0t7dpBfLzZm+i22+D33+2uTkQk/zgcZlHNwj4Kcguj1atX07dvXx588EGaNm1KrVq1OHDgQMF9wcuoX78+mzZtcroWGRl5zZ9Xp04dvL29WbNmTca1CxcuEBkZScOGDTOuVahQgUGDBjFz5kwmTpzoNHg4ICCA/v3789lnnzFnzhwiIiIyxr8UBM2pKiT168OaNfD++/DSS2Z/orAwmDgRBg4s2P/DiYjItalTpw4RERGsW7eOsmXL8v777xMbG+v0S70wjBw5kkcffZRWrVrRoUMH5syZw44dO6hVq9ZV3710thFAo0aNeOyxx3j66acpV64c1atX55133uHcuXM88sgjALz88su0bNmS0NBQkpOT+eGHHzK+7wkTJlClShWaNWuGh4cH//d//5exUn1BUWApRJ6e8PTT0Ls3DBoEmzbB4MHwzTcwZQq4ySxuEZFi46WXXuLw4cP07NmTkiVLMnToUPr160dcXFyh1vHAAw9w6NAhnnrqKZKSkrj33nsZNGhQtlaXnNx3333Zrh0+fJi33nqLtLQ0BgwYwNmzZ2nVqhU//vgjZcuWBcDb25sxY8Zw5MgR/Pz8uOmmm5g9ezYApUuX5u233+bAgQN4enrSunVrFi5cWKB7Szms6+kEcyHx8fEEBgYSFxdX6KveXouLF+G99+CVVyAlBcqUMeu4PPigWltExPWdO3eOPXv20LBhQ0rm5yASybVbbrmFypUr88U
|
||
|
|
"text/plain": [
|
||
|
|
"<Figure size 640x480 with 2 Axes>"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
"metadata": {},
|
||
|
|
"output_type": "display_data"
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"fig, ax = plt.subplots(2,1)\n",
|
||
|
|
"\n",
|
||
|
|
"xticks = np.arange(1, 6, step=1)\n",
|
||
|
|
"xlabels = [f'{x:1d}' for x in xticks]\n",
|
||
|
|
"ax[0].set_xticks(xticks, labels=xlabels)\n",
|
||
|
|
"ax[1].set_xticks(xticks, labels=xlabels)\n",
|
||
|
|
"\n",
|
||
|
|
"\n",
|
||
|
|
"ax[0].plot(history.history['loss'], color='b', label=\"Training Loss\")\n",
|
||
|
|
"legend = ax[0].legend(loc='best', shadow=True)\n",
|
||
|
|
"\n",
|
||
|
|
"ax[1].plot(history.history['accuracy'], color='b', label=\"Training Accuracy\")\n",
|
||
|
|
"legend = ax[1].legend(loc='best', shadow=True)\n"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "markdown",
|
||
|
|
"id": "ade9b5f7-9048-4b1d-876c-32bb16cfa7f7",
|
||
|
|
"metadata": {},
|
||
|
|
"source": [
|
||
|
|
"## upload model to datapool"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 8,
|
||
|
|
"id": "ae09ac93-5f47-4008-86d6-b9a503b0b20a",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [
|
||
|
|
{
|
||
|
|
"name": "stdout",
|
||
|
|
"output_type": "stream",
|
||
|
|
"text": [
|
||
|
|
"uploading file\n",
|
||
|
|
"<Response [201]> {\"PublicUrl\":\"https://share.storage.sandbox.iuk.hdm-stuttgart.de/upload/a0a1ca44-0413-4ad2-9031-78aac6336072/mymodel.keras\",\"Size\":97865925,\"Expiration\":\"2024-09-25T00:00:00Z\"}\n"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"source": [
|
||
|
|
"myurl = 'https://share.storage.sandbox.iuk.hdm-stuttgart.de/upload'\n",
|
||
|
|
"print(\"uploading file\")\n",
|
||
|
|
"files = {\n",
|
||
|
|
" 'fileUpload':('mymodel.keras', open('mymodel.keras', 'rb'),'application/octet-stream')\n",
|
||
|
|
"}\n",
|
||
|
|
"\n",
|
||
|
|
"response = requests.post(myurl, files=files)\n",
|
||
|
|
"print(response,response.text)"
|
||
|
|
]
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"cell_type": "code",
|
||
|
|
"execution_count": 9,
|
||
|
|
"id": "6db2bcf5-5c39-4796-b77e-1c16e3f42ac4",
|
||
|
|
"metadata": {},
|
||
|
|
"outputs": [],
|
||
|
|
"source": [
|
||
|
|
"K.clear_session()"
|
||
|
|
]
|
||
|
|
}
|
||
|
|
],
|
||
|
|
"metadata": {
|
||
|
|
"kernelspec": {
|
||
|
|
"display_name": "Python 3 (ipykernel)",
|
||
|
|
"language": "python",
|
||
|
|
"name": "python3"
|
||
|
|
},
|
||
|
|
"language_info": {
|
||
|
|
"codemirror_mode": {
|
||
|
|
"name": "ipython",
|
||
|
|
"version": 3
|
||
|
|
},
|
||
|
|
"file_extension": ".py",
|
||
|
|
"mimetype": "text/x-python",
|
||
|
|
"name": "python",
|
||
|
|
"nbconvert_exporter": "python",
|
||
|
|
"pygments_lexer": "ipython3",
|
||
|
|
"version": "3.11.7"
|
||
|
|
}
|
||
|
|
},
|
||
|
|
"nbformat": 4,
|
||
|
|
"nbformat_minor": 5
|
||
|
|
}
|