Compare commits
39 Commits
Author | SHA1 | Date |
---|---|---|
Cornelius Specht | 2443fcf2e3 | |
Cornelius Specht | e24b6eef5a | |
Cornelius Specht | 05291f41f6 | |
Malte Grosse | 7a3c6f8393 | |
Malte Grosse | 6e2731ea0f | |
Malte Grosse | 78c2e66855 | |
Malte Grosse | 6d896e921b | |
Malte Grosse | 67f6ecbbb9 | |
Malte Grosse | b3fdc6dcf6 | |
Malte Grosse | b966a664aa | |
Malte Grosse | cb1b4c6da1 | |
Malte Grosse | 547aca11bf | |
Malte Grosse | 9f1a7897cb | |
Malte Grosse | 9d75a10e93 | |
Malte Grosse | 83094016ba | |
Malte Grosse | 2701b15f28 | |
Malte Grosse | e55ab32bb1 | |
Malte Grosse | 147b8f63f5 | |
Malte Grosse | 02ea29fda0 | |
Malte Grosse | b84864841a | |
Malte Grosse | 511187ba16 | |
Malte Grosse | 45d58b7723 | |
Malte Grosse | c1c2795e42 | |
Malte Grosse | 9ffb755608 | |
Malte Grosse | f97151f785 | |
Malte Grosse | 8c03b5f688 | |
Malte Grosse | 2850c3ebd9 | |
Malte Grosse | 85f6d50cf7 | |
Malte Grosse | ca2071e7ca | |
Malte Grosse | 122c4ffe03 | |
Malte Grosse | 0d08778c74 | |
Malte Grosse | 01363eab00 | |
Malte Grosse | d7fc8fc080 | |
Malte Grosse | be73ea86b9 | |
Malte Grosse | 4603bb64a7 | |
Malte Grosse | 76b687bb9c | |
Malte Grosse | dc80437156 | |
Malte Grosse | 7a09a37cd4 | |
Malte Grosse | 88d5eff787 |
|
@ -1,32 +1,21 @@
|
||||||
platform: linux/amd64
|
matrix:
|
||||||
pipeline:
|
EPOCHS:
|
||||||
first-job:
|
- 20
|
||||||
image: busybox
|
- 30
|
||||||
commands:
|
OPTIMIZER:
|
||||||
- echo "ci working................. "
|
- adam
|
||||||
cpu:
|
- SGD
|
||||||
image: progrium/stress:latest
|
|
||||||
commands: /usr/bin/stress --cpu 20 --io 1 --vm 2 --vm-bytes 128M --timeout 90s
|
|
||||||
nvidia-test:
|
|
||||||
image: nvidia/cuda:11.6.2-base-ubuntu20.04
|
|
||||||
commands:
|
|
||||||
- nvidia-smi
|
|
||||||
environment:
|
|
||||||
- NVIDIA_VISIBLE_DEVICES=all
|
|
||||||
gpu:
|
|
||||||
image: oguzpastirmaci/gpu-burn:latest
|
|
||||||
environment:
|
|
||||||
- NVIDIA_VISIBLE_DEVICES=all
|
|
||||||
commands:
|
|
||||||
- cd /app
|
|
||||||
- ./gpu_burn 120
|
|
||||||
- echo "burned. done"
|
|
||||||
|
|
||||||
# 2nd:
|
|
||||||
# image: oguzpastirmaci/gpu-burn:latest
|
|
||||||
# environment:
|
|
||||||
# - NVIDIA_VISIBLE_DEVICES=all
|
|
||||||
# commands:
|
|
||||||
# - ./gpu_burn 60
|
|
||||||
|
|
||||||
|
steps:
|
||||||
|
"train":
|
||||||
|
image: nvcr.io/nvidia/tensorflow:23.10-tf2-py3
|
||||||
|
commands:
|
||||||
|
- echo "starting python script"
|
||||||
|
- python run.py
|
||||||
|
"compress and upload":
|
||||||
|
image: alpine:3
|
||||||
|
commands:
|
||||||
|
- apk --no-cache add zip curl
|
||||||
|
- zip mymodel.zip mymodel.keras
|
||||||
|
- curl -F fileUpload=@mymodel.zip https://share.storage.sandbox.iuk.hdm-stuttgart.de/upload
|
||||||
|
|
||||||
|
|
|
@ -0,0 +1,93 @@
|
||||||
|
import tensorflow as tf
|
||||||
|
from tensorflow import keras
|
||||||
|
import requests
|
||||||
|
import numpy as np
|
||||||
|
import os
|
||||||
|
# Version Information
|
||||||
|
# tensorflow 2.2.0 , Cudnn7.6.5 and Cuda 10.1 , python 3.8
|
||||||
|
from keras import backend as K
|
||||||
|
K.clear_session()
|
||||||
|
|
||||||
|
EPOCHS = int(os.getenv("EPOCHS", default = 10))
|
||||||
|
OPTIMIZER = os.getenv("OPTIMIZER", default = "SGD")
|
||||||
|
|
||||||
|
|
||||||
|
gpus = tf.config.experimental.list_physical_devices('GPU')
|
||||||
|
# if gpus:
|
||||||
|
# try:
|
||||||
|
# tf.config.experimental.set_virtual_device_configuration(gpus[0], [tf.config.experimental.VirtualDeviceConfiguration(memory_limit=6024)])
|
||||||
|
# except RuntimeError as e:
|
||||||
|
# print(e)
|
||||||
|
# os.exit(1)
|
||||||
|
|
||||||
|
print(tf.config.experimental.list_physical_devices())
|
||||||
|
|
||||||
|
print(tf.__version__)
|
||||||
|
|
||||||
|
print(tf.test.is_built_with_cuda())
|
||||||
|
|
||||||
|
(X_train, y_train), (X_test,y_test) = tf.keras.datasets.cifar10.load_data()
|
||||||
|
|
||||||
|
(X_train, y_train), (X_test,y_test) = tf.keras.datasets.cifar10.load_data()
|
||||||
|
|
||||||
|
print(X_train.shape,y_train.shape)
|
||||||
|
|
||||||
|
classes = ["airplane","automobile","bird","cat","deer","dog","frog","horse","ship","truck"]
|
||||||
|
|
||||||
|
print(classes[y_train[3][0]])
|
||||||
|
|
||||||
|
print("pre processing: scale images")
|
||||||
|
|
||||||
|
X_train_scaled = X_train / 255
|
||||||
|
X_test_scaled = X_test / 255
|
||||||
|
|
||||||
|
y_train_categorical = keras.utils.to_categorical(
|
||||||
|
y_train, num_classes=10, dtype='float32'
|
||||||
|
)
|
||||||
|
y_test_categorical = keras.utils.to_categorical(
|
||||||
|
y_test, num_classes=10, dtype='float32'
|
||||||
|
)
|
||||||
|
|
||||||
|
print("model build")
|
||||||
|
|
||||||
|
|
||||||
|
def get_model():
|
||||||
|
model = keras.Sequential([
|
||||||
|
keras.layers.Flatten(input_shape=(32,32,3)),
|
||||||
|
keras.layers.Dense(3000, activation='relu'),
|
||||||
|
keras.layers.Dense(1000, activation='relu'),
|
||||||
|
keras.layers.Dense(10, activation='sigmoid')
|
||||||
|
])
|
||||||
|
|
||||||
|
model.compile(optimizer='SGD',
|
||||||
|
loss='categorical_crossentropy',
|
||||||
|
metrics=['accuracy'])
|
||||||
|
return model
|
||||||
|
|
||||||
|
with tf.device('/GPU:0'):
|
||||||
|
model = keras.Sequential([
|
||||||
|
keras.layers.Flatten(input_shape=(32,32,3)),
|
||||||
|
keras.layers.Dense(3000, activation='relu'),
|
||||||
|
keras.layers.Dense(1000, activation='relu'),
|
||||||
|
keras.layers.Dense(10, activation='sigmoid')
|
||||||
|
])
|
||||||
|
# g
|
||||||
|
model.compile(optimizer=OPTIMIZER,
|
||||||
|
loss='categorical_crossentropy',
|
||||||
|
metrics=['accuracy'])
|
||||||
|
model.fit(X_train_scaled, y_train_categorical, epochs=EPOCHS)
|
||||||
|
model.save('mymodel.keras')
|
||||||
|
|
||||||
|
print("finished training")
|
||||||
|
|
||||||
|
myurl = 'https://share.storage.sandbox.iuk.hdm-stuttgart.de/upload'
|
||||||
|
|
||||||
|
print("uploading file")
|
||||||
|
files = {
|
||||||
|
'fileUpload':('mymodel.keras', open('mymodel.keras', 'rb'),'application/octet-stream')
|
||||||
|
}
|
||||||
|
|
||||||
|
response = requests.post(myurl, files=files)
|
||||||
|
print(response,response.text)
|
||||||
|
|
||||||
|
print("done")
|
Loading…
Reference in New Issue