From ff67f2185826d3fe8722ffaaa4eb680f70cbf3a9 Mon Sep 17 00:00:00 2001 From: Derek Holloway Date: Thu, 26 Feb 2026 21:39:57 -0800 Subject: [PATCH] Make slightly more memory efficient --- WebServer/AIPython/features.py | 108 ++++++++++++++++----------------- 1 file changed, 51 insertions(+), 57 deletions(-) diff --git a/WebServer/AIPython/features.py b/WebServer/AIPython/features.py index 82f79f10..7ed7b957 100644 --- a/WebServer/AIPython/features.py +++ b/WebServer/AIPython/features.py @@ -1,14 +1,46 @@ from sklearn.preprocessing import MinMaxScaler -import pandas as pd import numpy as np -def MakeFeatures(df, i): - # Remove the ticker column - df.columns = df.columns.get_level_values(0) - - # Make sure Date is a number object - df = df.reset_index() - df['Date'] = pd.to_numeric(pd.to_datetime(df['Date'])) +def MakeFeatures(df): + # Convert all F64 to F32 to save ram + df = df.astype({col: 'float32' for col in df.select_dtypes('float64').columns}) + + # Candle Wick's + df['Spread'] = (df['High'] - df['Low']) / df['Close'] # in percentage of price + candle_top = df[['Open', 'Close']].max(axis=1) + df['Body_Size'] = (df['Close'] - df['Open']).abs() / (df['High'] - df['Low']) + df['Upper_Shadow'] = (df['High'] - candle_top) / (df['High'] - df['Low']) + +# Remove Unused Symbols to save ram + df.drop('Open', axis=1, inplace=True) + df.drop('High', axis=1, inplace=True) + df.drop('Low', axis=1, inplace=True) + + # Is volume 2x higher than the 20-day average? + df['Vol_Intensity'] = df['Volume'] / df['Volume'].rolling(20).mean() + # Volume Change + df['Volume_Chg'] = df['Volume'].pct_change() + +# Remove Unused Symbols to save ram + df.drop('Volume', axis=1, inplace=True) + + # Moving Average Crossover (Golden/Death Cross logic) + df['Moving_Average_5'] = df['Close'].rolling(window=20).mean() + df['Moving_Average_20'] = df['Close'].rolling(window=20).mean() + # if short term > long term (bullish), else 0 + df['Trend_Signal'] = (df['Moving_Average_5'] > df['Moving_Average_20']).astype(int) + # Distance from MA (How overextended are we?) + df['Dist_From_MA20'] = (df['Close'] / df['Moving_Average_20']) - 1 + + # Bollinger Band Position (Where are we relative to volatility?) + std_20 = df['Close'].rolling(20).std() + upper_band = df['Moving_Average_20'] + (std_20 * 2) + lower_band = df['Moving_Average_20'] - (std_20 * 2) + df['BB_Pos'] = (df['Close'] - lower_band) / (upper_band - lower_band) + +# Remove Unused Symbols to save ram + df.drop('Moving_Average_5', axis=1, inplace=True) + df.drop('Moving_Average_20', axis=1, inplace=True) # Add feature for Returns df['Return'] = df['Close'].pct_change() @@ -27,31 +59,6 @@ def MakeFeatures(df, i): rs = gain / loss df['RSI'] = 100 - (100 / (1 + rs)) - # Moving Average Crossover (Golden/Death Cross logic) - df['Moving_Average_5'] = df['Close'].rolling(window=20).mean() - df['Moving_Average_20'] = df['Close'].rolling(window=20).mean() - # if short term > long term (bullish), else 0 - df['Trend_Signal'] = (df['Moving_Average_5'] > df['Moving_Average_20']).astype(int) - # Distance from MA (How overextended are we?) - df['Dist_From_MA20'] = (df['Close'] / df['Moving_Average_20']) - 1 - - # Bollinger Band Position (Where are we relative to volatility?) - std_20 = df['Close'].rolling(20).std() - upper_band = df['Moving_Average_20'] + (std_20 * 2) - lower_band = df['Moving_Average_20'] - (std_20 * 2) - df['BB_Pos'] = (df['Close'] - lower_band) / (upper_band - lower_band) - - # Candle Wick's - df['Spread'] = (df['High'] - df['Low']) / df['Close'] # in percentage of price - candle_top = df[['Open', 'Close']].max(axis=1) - df['Body_Size'] = (df['Close'] - df['Open']).abs() / (df['High'] - df['Low']) - df['Upper_Shadow'] = (df['High'] - candle_top) / (df['High'] - df['Low']) - - # Is volume 2x higher than the 20-day average? - df['Vol_Intensity'] = df['Volume'] / df['Volume'].rolling(20).mean() - # Volume Change - df['Volume_Chg'] = df['Volume'].pct_change() - # Return lagged for lag in range(1, 4): df[f'Return_Lag_{lag}'] = df['Return'].shift(lag) @@ -61,14 +68,17 @@ def MakeFeatures(df, i): df['Target_Close'] = np.log(df['Close'].shift(-5) / df['Close']) df['Target_Close'] = np.log(df['Close'].shift(-5) / df['Close']) / df['Volatility_20'] - # Remove noise data from the model to really focus on percent changes - df.drop('Open', axis=1) - df.drop('High', axis=1) - df.drop('Low', axis=1) - df.drop('Close', axis=1) - df.drop('Volume', axis=1) - df.drop('Moving_Average_5', axis=1) - df.drop('Moving_Average_20', axis=1) +# Remove Unused Symbols to save ram + df.drop('Close', axis=1, inplace=True) + + # Drop rows with null values + df.dropna(inplace=True) + + # Replace Infinity with 0 -> This fixes the AI mental breakdown + df = df.replace([np.inf, -np.inf], 0) + + # Replace Infinity with 0 -> This fixes the AI mental breakdown + df['Volume_Chg'] = df['Volume_Chg'].replace([np.inf, -np.inf], 0) # Return new df with new features return df @@ -90,19 +100,3 @@ def Prepare(df): y_scaled = target_scaler.fit_transform(Y.values.reshape(-1, 1)) return X_scaled, y_scaled, feature_scaler, target_scaler - -def CleanDF(df): - # Make date the index so it doesnt influence the training - df.set_index('Date', inplace=True) - - # Drop rows with null values - df.dropna(inplace=True) - - # Replace Infinity with 0 -> This fixes the AI mental breakdown - df = df.replace([np.inf, -np.inf], 0) - - # Replace Infinity with 0 -> This fixes the AI mental breakdown - df['Volume_Chg'] = df['Volume_Chg'].replace([np.inf, -np.inf], 0) - - # Return new df thats cleaned - return df