Work on fine-tuning and Important Feature selections

WebServer/AIPython/ai-trainer.py

@@ -8,6 +8,7 @@ import os
 os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
 from sklearn.model_selection import train_test_split
 from keras import Sequential, layers, optimizers, losses
+from keras.callbacks import ReduceLROnPlateau
 
 def TrainAI(include_pull):
 
@@ -35,36 +36,43 @@ def TrainAI(include_pull):
     # Split out the test and train
     train_features, test_features, train_labels, test_labels = train_test_split(X, Y, test_size=0.2)
 
-    # Create a normalizer to nomralize the data
-    normalizer = layers.Normalization(axis=-1)
-    normalizer.adapt(np.array(train_features))
-
     # Create the DNN
     dnn_model = Sequential([
         layers.Input(shape=(train_features.shape[1],)), # Load the feature count dynamically
-        normalizer,
-        layers.Dense(64, activation='elu'),
-        layers.Dense(64, activation='elu'),
-        layers.Dense(1)
+        layers.Dense(256, activation='elu'),
+        layers.Dropout(0.1),                    # Small dropout to prevent it from memorizing noise
+        layers.Dense(128, activation='elu'),
+        layers.Dropout(0.1),                    # Small dropout to prevent it from memorizing noise
+        layers.Dense(24, activation='elu'),
+        layers.Dense(1, activation='linear')
     ])
 
+    # Allow negative numbers
+    dnn_model.add(layers.LeakyReLU(alpha=0.01))
+
     # Configure the model
     dnn_model.compile(
-        optimizer=optimizers.Adam(learning_rate=0.00001, clipvalue=1.0),
-        loss=losses.Huber()
+        optimizer=optimizers.Adam(learning_rate=0.0001, clipvalue=1.0),
+        loss="mse",
+        metrics=['mae']             # See it train while it runs
     )
 
     # Show the summary before training the model
     dnn_model.summary()
 
+    # Learning rate reducer
+    reduce_lr = ReduceLROnPlateau(monitor='val_loss', factor=0.5, patience=5, min_lr=0.0000001)
+
     # Train the model
     Training_Data = dnn_model.fit( 
         train_features,
         train_labels,
         batch_size=64,
-        epochs=39,                  # Tuned to the point before overfitting
+        epochs=50,                  # Tuned to the point before overfitting
         verbose=1,                  # Show progress
-        validation_split = 0.2      # Calculate validation results on 20% of the training data.
+        validation_split = 0.2,     # Calculate validation results on 20% of the training data.
+        shuffle=True,
+        callbacks=[reduce_lr]       # Reduce the learning_rate every run
     )
 
     # Predict
@@ -79,7 +87,7 @@ def TrainAI(include_pull):
     _ = plt.plot(lims, lims)
 
 
-    # Current Test Results: 1.221876300405711e-05
+    # Current Test Results: 0.3382711112499237
     test_results = dnn_model.evaluate(
         test_features, test_labels, verbose=0
     )
@@ -89,4 +97,6 @@ def TrainAI(include_pull):
     dnn_model.save(os.path.join(DATA_DIR, "model.keras"))
 
 if __name__ == "__main__":
-    TrainAI(False)
+    TrainAI(True)
+
+# Last train Predicted Target_Close: [0.0022113274317234755, 0.0021446370519697666, 0.0022628342267125845, 0.002175702480599284, 0.0021452796645462513, 0.0020838389173150063, 0.0017336219316348433, 0.002210840117186308, 0.0021144403144717216, 0.0021278387866914272, 0.0021266420371830463, 0.002261851681396365, 0.002108299173414707, 0.002121902070939541, 0.0022294146474450827]

WebServer/AIPython/features.py

@@ -1,4 +1,4 @@
-from sklearn.preprocessing import StandardScaler
+from sklearn.preprocessing import MinMaxScaler
 import pandas as pd
 import numpy as np
 
@@ -10,18 +10,13 @@ def MakeFeatures(df, i):
     df = df.reset_index()
     df['Date'] = pd.to_numeric(pd.to_datetime(df['Date']))
 
-    # Add the Symbol column for tracking | as an int 1 hot encoded
-    df['Symbol'] = i
-
-    # Add feature Spread
-    df['Spread'] = abs( df['High'] - df['Low'] )
-
     # Add feature for Returns
     df['Return'] = df['Close'].pct_change()
+    # Log Returns (Better for AI than pct_change for statistical normality)
+    df['Log_Return'] = np.log(df['Close'] / df['Close'].shift(1))
 
     # Add feature for volitility last 5
     df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std())
-
     # Add feature for volitility last 20
     df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std())
 
@@ -33,11 +28,10 @@ def MakeFeatures(df, i):
     df['RSI'] = 100 - (100 / (1 + rs))
 
     # Moving Average Crossover (Golden/Death Cross logic)
-    df['Moving_Average_5'] = df['Close'].rolling(window=5).mean()
+    df['Moving_Average_5'] = df['Close'].rolling(window=20).mean()
     df['Moving_Average_20'] = df['Close'].rolling(window=20).mean()
     # if short term > long term (bullish), else 0
     df['Trend_Signal'] = (df['Moving_Average_5'] > df['Moving_Average_20']).astype(int)
-
     # Distance from MA (How overextended are we?)
     df['Dist_From_MA20'] = (df['Close'] / df['Moving_Average_20']) - 1
 
@@ -47,30 +41,52 @@ def MakeFeatures(df, i):
     lower_band = df['Moving_Average_20'] - (std_20 * 2)
     df['BB_Pos'] = (df['Close'] - lower_band) / (upper_band - lower_band)
 
-    # Log Returns (Better for AI than pct_change for statistical normality)
-    df['Log_Return'] = np.log(df['Close'] / df['Close'].shift(1))
-
-    # This is our training metric of 5 days ahead
-    df['Target_Close'] = df['Close'].shift(-5).pct_change()
+    # Candle Wick's
+    df['Spread'] = (df['High'] - df['Low']) / df['Close'] # in percentage of price
+    candle_top = df[['Open', 'Close']].max(axis=1)
+    df['Body_Size'] = (df['Close'] - df['Open']).abs() / (df['High'] - df['Low'])
+    df['Upper_Shadow'] = (df['High'] - candle_top) / (df['High'] - df['Low'])
 
+    # Is volume 2x higher than the 20-day average?
+    df['Vol_Intensity'] = df['Volume'] / df['Volume'].rolling(20).mean()
     # Volume Change
     df['Volume_Chg'] = df['Volume'].pct_change()
 
+    # Return lagged
+    for lag in range(1, 4):
+        df[f'Return_Lag_{lag}'] = df['Return'].shift(lag)
+        df[f'Vol_Lag_{lag}'] = df['Volume_Chg'].shift(lag)
+
+    # This is our training metric of price difference 5 days ahead
+    df['Target_Close'] = np.log(df['Close'].shift(-5) / df['Close'])
+    df['Target_Close'] = np.log(df['Close'].shift(-5) / df['Close']) / df['Volatility_20']
+
+    # Remove noise data from the model to really focus on percent changes
+    df.drop('Open', axis=1)
+    df.drop('High', axis=1)
+    df.drop('Low', axis=1)
+    df.drop('Close', axis=1)
+    df.drop('Volume', axis=1)
+    df.drop('Moving_Average_5', axis=1)
+    df.drop('Moving_Average_20', axis=1)
+
     # Return new df with new features
     return df
 
 def Prepare(df):
 
+    df = df.replace([np.inf, -np.inf], 0)
+
     # Remove indicators and set the target
     X = df.drop('Target_Close', axis=1)
     Y = df['Target_Close']
 
     # Scale the features to the same size
-    feature_scaler = StandardScaler()
+    feature_scaler = MinMaxScaler()
     X_scaled = feature_scaler.fit_transform(X)
 
     # Safe for the Y
-    target_scaler = StandardScaler()
+    target_scaler = MinMaxScaler()
     y_scaled = target_scaler.fit_transform(Y.values.reshape(-1, 1))
 
     return X_scaled, y_scaled, feature_scaler, target_scaler
@@ -82,6 +98,9 @@ def CleanDF(df):
     # Drop rows with null values
     df.dropna(inplace=True)
 
+    # Replace Infinity with 0 -> This fixes the AI mental breakdown
+    df = df.replace([np.inf, -np.inf], 0)
+
     # Replace Infinity with 0 -> This fixes the AI mental breakdown
     df['Volume_Chg'] = df['Volume_Chg'].replace([np.inf, -np.inf], 0)