diff --git a/Training_Results.png b/Training_Results.png new file mode 100644 index 00000000..6b10b314 Binary files /dev/null and b/Training_Results.png differ diff --git a/WebServer/AIPython/Target_Close_Average.txt b/WebServer/AIPython/Target_Close_Average.txt new file mode 100644 index 00000000..8edbada3 --- /dev/null +++ b/WebServer/AIPython/Target_Close_Average.txt @@ -0,0 +1 @@ +0.198600759951581 \ No newline at end of file diff --git a/WebServer/AIPython/aitrainer.py b/WebServer/AIPython/aitrainer.py index ca3e8639..83877e3c 100644 --- a/WebServer/AIPython/aitrainer.py +++ b/WebServer/AIPython/aitrainer.py @@ -1,8 +1,19 @@ import pandas as pd +from sklearn.metrics import f1_score, precision_score, recall_score, accuracy_score import features import joblib import os + +# Suppress TensorFlow INFO and WARNING logs +os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' + +# Suppress specialized XLA and autotuning logs +os.environ['TF_XLA_FLAGS'] = '--tf_xla_enable_xla_devices=false' +os.environ['TF_CPP_MAX_VLOG_LEVEL'] = '0' + +# CPU only os.environ["CUDA_VISIBLE_DEVICES"] = "-1" + from sklearn.model_selection import train_test_split from keras import Sequential, layers, optimizers from keras.callbacks import ReduceLROnPlateau @@ -19,6 +30,9 @@ def TrainAI(): # Use external featuers to make sure loaded is the same dataset = features.MakeFeatures(dataset) + # Make sure empty columns dont exist + dataset = dataset.dropna() + # Create the X, Y vareables X, Y, X_Scaler, Y_Scaler = features.Prepare(dataset) @@ -30,7 +44,7 @@ def TrainAI(): print(dataset.dtypes) # Split out the test and train - train_features, test_features, train_labels, test_labels = train_test_split(X, Y, test_size=0.2) + train_features, test_features, train_labels, test_labels = train_test_split(X, Y, test_size=0.2, shuffle=False) # Keep the training and test data in order as its sequential # Create the DNN dnn_model = Sequential([ @@ -45,9 +59,6 @@ def TrainAI(): layers.Dense(1, activation='linear') # DNN layer ]) - # Allow negative numbers - dnn_model.add(layers.LeakyReLU(alpha=0.01)) - # Configure the model dnn_model.compile( optimizer=optimizers.Adam(learning_rate=0.0001, clipvalue=1.0), @@ -59,7 +70,7 @@ def TrainAI(): dnn_model.summary() # Learning rate reducer - reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.0000001) + reduce_lr = ReduceLROnPlateau(monitor='loss', factor=0.5, patience=5, min_lr=0.0000001) # Train the model Training_Data = dnn_model.fit( @@ -68,8 +79,7 @@ def TrainAI(): batch_size=64, epochs=50, # Tuned to the point before overfitting verbose=1, # Show progress - validation_split = 0.2, # Calculate validation results on 20% of the training data. - shuffle=True, + shuffle=False, # Time series data callbacks=[reduce_lr] # Reduce the learning_rate every run ) @@ -78,6 +88,27 @@ def TrainAI(): test_features, test_labels, verbose=0 ) + # Perform test on test data split earlier + predictions = dnn_model.predict(test_features) + + # Convert to Binary Direction (The "Signal") + y_true_binary = (test_labels > 0).astype(int) + y_pred_binary = (predictions > 0).astype(int) + + # Calculate the metrics + acc = accuracy_score(y_true_binary, y_pred_binary) + prec = precision_score(y_true_binary, y_pred_binary) + rec = recall_score(y_true_binary, y_pred_binary) + f1 = f1_score(y_true_binary, y_pred_binary) + + # Output the meterics; this gets spit out when training from the python file directly. The C# interop does not output these + print(f"Test Loss: {test_results[0]:.4f}") + print(f"Test MAE: {test_results[1]:.2%}") + print(f"Test Accuracy: {acc:.2%}") + print(f"Test Precision: {prec:.2%}") + print(f"Test Recall: {rec:.2%}") + print(f"F1 Score: {f1:.2%}") + # Save the model dnn_model.save(os.path.join(DATA_DIR, "model.keras")) diff --git a/WebServer/AIPython/data/feature_scaler.pkl b/WebServer/AIPython/data/feature_scaler.pkl index af4c0dd9..393b48af 100644 Binary files a/WebServer/AIPython/data/feature_scaler.pkl and b/WebServer/AIPython/data/feature_scaler.pkl differ diff --git a/WebServer/AIPython/data/model.keras b/WebServer/AIPython/data/model.keras index f2915421..e641570a 100644 Binary files a/WebServer/AIPython/data/model.keras and b/WebServer/AIPython/data/model.keras differ diff --git a/WebServer/AIPython/data/target_scaler.pkl b/WebServer/AIPython/data/target_scaler.pkl index 8e881bde..6f37ec6d 100644 Binary files a/WebServer/AIPython/data/target_scaler.pkl and b/WebServer/AIPython/data/target_scaler.pkl differ