Make slightly more memory efficient

WebServer/AIPython/features.py

@@ -1,14 +1,46 @@
 from sklearn.preprocessing import MinMaxScaler
-import pandas as pd
 import numpy as np
 
-def MakeFeatures(df, i):
+def MakeFeatures(df):
-    # Remove the ticker column
+    # Convert all F64 to F32 to save ram
-    df.columns = df.columns.get_level_values(0)
+    df = df.astype({col: 'float32' for col in df.select_dtypes('float64').columns})
 
-    # Make sure Date is a number object
+    # Candle Wick's
-    df = df.reset_index()
+    df['Spread'] = (df['High'] - df['Low']) / df['Close'] # in percentage of price
-    df['Date'] = pd.to_numeric(pd.to_datetime(df['Date']))
+    candle_top = df[['Open', 'Close']].max(axis=1)
+    df['Body_Size'] = (df['Close'] - df['Open']).abs() / (df['High'] - df['Low'])
+    df['Upper_Shadow'] = (df['High'] - candle_top) / (df['High'] - df['Low'])
+
+# Remove Unused Symbols to save ram
+    df.drop('Open', axis=1, inplace=True)
+    df.drop('High', axis=1, inplace=True)
+    df.drop('Low', axis=1, inplace=True)
+
+     # Is volume 2x higher than the 20-day average?
+    df['Vol_Intensity'] = df['Volume'] / df['Volume'].rolling(20).mean()
+    # Volume Change
+    df['Volume_Chg'] = df['Volume'].pct_change()
+
+# Remove Unused Symbols to save ram
+    df.drop('Volume', axis=1, inplace=True)
+
+    # Moving Average Crossover (Golden/Death Cross logic)
+    df['Moving_Average_5'] = df['Close'].rolling(window=20).mean()
+    df['Moving_Average_20'] = df['Close'].rolling(window=20).mean()
+    # if short term > long term (bullish), else 0
+    df['Trend_Signal'] = (df['Moving_Average_5'] > df['Moving_Average_20']).astype(int)
+    # Distance from MA (How overextended are we?)
+    df['Dist_From_MA20'] = (df['Close'] / df['Moving_Average_20']) - 1
+
+    # Bollinger Band Position (Where are we relative to volatility?)
+    std_20 = df['Close'].rolling(20).std()
+    upper_band = df['Moving_Average_20'] + (std_20 * 2)
+    lower_band = df['Moving_Average_20'] - (std_20 * 2)
+    df['BB_Pos'] = (df['Close'] - lower_band) / (upper_band - lower_band)
+
+# Remove Unused Symbols to save ram
+    df.drop('Moving_Average_5', axis=1, inplace=True)
+    df.drop('Moving_Average_20', axis=1, inplace=True)
 
     # Add feature for Returns
     df['Return'] = df['Close'].pct_change()
@@ -27,31 +59,6 @@ def MakeFeatures(df, i):
     rs = gain / loss
     df['RSI'] = 100 - (100 / (1 + rs))
 
-    # Moving Average Crossover (Golden/Death Cross logic)
-    df['Moving_Average_5'] = df['Close'].rolling(window=20).mean()
-    df['Moving_Average_20'] = df['Close'].rolling(window=20).mean()
-    # if short term > long term (bullish), else 0
-    df['Trend_Signal'] = (df['Moving_Average_5'] > df['Moving_Average_20']).astype(int)
-    # Distance from MA (How overextended are we?)
-    df['Dist_From_MA20'] = (df['Close'] / df['Moving_Average_20']) - 1
-
-    # Bollinger Band Position (Where are we relative to volatility?)
-    std_20 = df['Close'].rolling(20).std()
-    upper_band = df['Moving_Average_20'] + (std_20 * 2)
-    lower_band = df['Moving_Average_20'] - (std_20 * 2)
-    df['BB_Pos'] = (df['Close'] - lower_band) / (upper_band - lower_band)
-
-    # Candle Wick's
-    df['Spread'] = (df['High'] - df['Low']) / df['Close'] # in percentage of price
-    candle_top = df[['Open', 'Close']].max(axis=1)
-    df['Body_Size'] = (df['Close'] - df['Open']).abs() / (df['High'] - df['Low'])
-    df['Upper_Shadow'] = (df['High'] - candle_top) / (df['High'] - df['Low'])
-
-    # Is volume 2x higher than the 20-day average?
-    df['Vol_Intensity'] = df['Volume'] / df['Volume'].rolling(20).mean()
-    # Volume Change
-    df['Volume_Chg'] = df['Volume'].pct_change()
-
     # Return lagged
     for lag in range(1, 4):
         df[f'Return_Lag_{lag}'] = df['Return'].shift(lag)
@@ -61,14 +68,17 @@ def MakeFeatures(df, i):
     df['Target_Close'] = np.log(df['Close'].shift(-5) / df['Close'])
     df['Target_Close'] = np.log(df['Close'].shift(-5) / df['Close']) / df['Volatility_20']
 
-    # Remove noise data from the model to really focus on percent changes
+# Remove Unused Symbols to save ram
-    df.drop('Open', axis=1)
+    df.drop('Close', axis=1, inplace=True)
-    df.drop('High', axis=1)
+
-    df.drop('Low', axis=1)
+    # Drop rows with null values
-    df.drop('Close', axis=1)
+    df.dropna(inplace=True)
-    df.drop('Volume', axis=1)
+
-    df.drop('Moving_Average_5', axis=1)
+    # Replace Infinity with 0 -> This fixes the AI mental breakdown
-    df.drop('Moving_Average_20', axis=1)
+    df = df.replace([np.inf, -np.inf], 0)
+
+    # Replace Infinity with 0 -> This fixes the AI mental breakdown
+    df['Volume_Chg'] = df['Volume_Chg'].replace([np.inf, -np.inf], 0)
 
     # Return new df with new features
     return df
@@ -90,19 +100,3 @@ def Prepare(df):
     y_scaled = target_scaler.fit_transform(Y.values.reshape(-1, 1))
 
     return X_scaled, y_scaled, feature_scaler, target_scaler
-
-def CleanDF(df):
-    # Make date the index so it doesnt influence the training
-    df.set_index('Date', inplace=True)
-
-    # Drop rows with null values
-    df.dropna(inplace=True)
-
-    # Replace Infinity with 0 -> This fixes the AI mental breakdown
-    df = df.replace([np.inf, -np.inf], 0)
-
-    # Replace Infinity with 0 -> This fixes the AI mental breakdown
-    df['Volume_Chg'] = df['Volume_Chg'].replace([np.inf, -np.inf], 0)
-
-    # Return new df thats cleaned
-    return df