Move away from Python Interop for better stability

WebServer/AIPython/aitrainer.py

@@ -0,0 +1,83 @@
+import pandas as pd
+import features
+import joblib
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
+from sklearn.model_selection import train_test_split
+from keras import Sequential, layers, optimizers
+from keras.callbacks import ReduceLROnPlateau
+
+def TrainAI():
+
+    # Get the CWD for pathing due to being called from C#
+    SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+    DATA_DIR = os.path.join(SCRIPT_DIR, "data")
+
+    # Load the dataset
+    dataset = pd.read_parquet(os.path.join(DATA_DIR, "stocks.parquet"))
+
+    # Use external featuers to make sure loaded is the same
+    dataset = features.MakeFeatures(dataset)
+
+    # Create the X, Y vareables
+    X, Y, X_Scaler, Y_Scaler = features.Prepare(dataset)
+
+    # Save the scalers for future use
+    joblib.dump(X_Scaler, os.path.join(DATA_DIR, "feature_scaler.pkl"))
+    joblib.dump(Y_Scaler, os.path.join(DATA_DIR, "target_scaler.pkl"))
+
+    # Show the datatypes
+    print(dataset.dtypes)
+
+    # Split out the test and train
+    train_features, test_features, train_labels, test_labels = train_test_split(X, Y, test_size=0.2)
+
+    # Create the DNN
+    dnn_model = Sequential([
+        layers.Input(shape=(train_features.shape[1],)), # Load the feature count dynamically
+        layers.Dense(256, activation='elu'),
+        layers.Dropout(0.1),                    # Small dropout to prevent it from memorizing noise
+        layers.Dense(128, activation='elu'),
+        layers.Dropout(0.1),                    # Small dropout to prevent it from memorizing noise
+        layers.Dense(24, activation='elu'),
+        layers.Dense(1, activation='linear')
+    ])
+
+    # Allow negative numbers
+    dnn_model.add(layers.LeakyReLU(alpha=0.01))
+
+    # Configure the model
+    dnn_model.compile(
+        optimizer=optimizers.Adam(learning_rate=0.0001, clipvalue=1.0),
+        loss="mse",
+        metrics=['mae']             # See it train while it runs
+    )
+
+    # Show the summary before training the model
+    dnn_model.summary()
+
+    # Learning rate reducer
+    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.0000001)
+
+    # Train the model
+    Training_Data = dnn_model.fit( 
+        train_features,
+        train_labels,
+        batch_size=64,
+        epochs=50,                  # Tuned to the point before overfitting
+        verbose=1,                  # Show progress
+        validation_split = 0.2,     # Calculate validation results on 20% of the training data.
+        shuffle=True,
+        callbacks=[reduce_lr]       # Reduce the learning_rate every run
+    )
+
+    # Current Test Results: 0.3382711112499237
+    test_results = dnn_model.evaluate(
+        test_features, test_labels, verbose=0
+    )
+
+    # Save the model
+    dnn_model.save(os.path.join(DATA_DIR, "model.keras"))
+
+if __name__ == "__main__":
+    TrainAI()