From 1343e93a597019d9fb0f46b8f694e88500c47bd3 Mon Sep 17 00:00:00 2001 From: Derek Holloway Date: Sun, 15 Feb 2026 21:24:05 -0800 Subject: [PATCH] Create a python interop service and wire it up --- WebServer/AIPython/datapuller.py | 81 +++++++++++++------------- WebServer/Controllers/PythonInterop.cs | 36 ++++++++++++ WebServer/Program.cs | 12 +++- WebServer/WebServer.csproj | 10 ++++ 4 files changed, 97 insertions(+), 42 deletions(-) create mode 100644 WebServer/Controllers/PythonInterop.cs diff --git a/WebServer/AIPython/datapuller.py b/WebServer/AIPython/datapuller.py index 78f7120a..333ef732 100644 --- a/WebServer/AIPython/datapuller.py +++ b/WebServer/AIPython/datapuller.py @@ -1,53 +1,56 @@ +import os import yfinance as yf import pandas as pd -class DataPuller: +def pull(): - @staticmethod - def pull(): - # Import the S&P 500 symbols - symbols = pd.read_excel("./data/stock_symbols.xlsx") - symbols.columns = symbols.columns.str.strip() - tickers = symbols['Symbol'].tolist() + # Get the CWD for pathing due to being called from C# now + SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) + DATA_DIR = os.path.join(SCRIPT_DIR, "data") - # Scrape the data - all_data = [] - for i, symbol in enumerate(tickers): # Try first 20 - print(f"Processing: {i} of {len(tickers)}") - df = yf.download(symbol, period="max", auto_adjust=True) - if not df.empty: - # Remove the ticker column - df.columns = df.columns.get_level_values(0) - - # Make sure Date is actually a Date Object - df = df.reset_index() - df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d") - df.set_index('Date', inplace=True) + # Import the S&P 500 symbols + symbols = pd.read_excel(os.path.join(DATA_DIR, "stock_symbols.xlsx")) + symbols.columns = symbols.columns.str.strip() + tickers = symbols['Symbol'].tolist() - # Add the Symbol column for tracking - df['Symbol'] = symbol + # Scrape the data + all_data = [] + for i, symbol in enumerate(tickers): # Try first 20 + print(f"Processing: {i} of {len(tickers)}") + df = yf.download(symbol, period="max", auto_adjust=True) + if not df.empty: + # Remove the ticker column + df.columns = df.columns.get_level_values(0) - # Add feature Spread - df['Spread'] = abs( df['High'] - df['Low'] ) + # Make sure Date is actually a Date Object + df = df.reset_index() + df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d") + df.set_index('Date', inplace=True) - # Add feature for Returns - df['Return'] = df['Close'].pct_change() + # Add the Symbol column for tracking + df['Symbol'] = symbol + + # Add feature Spread + df['Spread'] = abs( df['High'] - df['Low'] ) - # Add feature for volitility last 5 - df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std()) + # Add feature for Returns + df['Return'] = df['Close'].pct_change() - # Add feature for volitility last 20 - df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std()) + # Add feature for volitility last 5 + df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std()) - all_data.append(df) + # Add feature for volitility last 20 + df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std()) - # Concatinate into a combined list and cache - print("Processing data") - final_df = pd.concat(all_data) + all_data.append(df) - # Drop rows with null values - final_df.dropna(inplace=True) + # Concatinate into a combined list and cache + print("Processing data") + final_df = pd.concat(all_data) - print("Writing data to file") - final_df.to_parquet("./data/stocks.parquet") - final_df.head(200).to_csv("./data/stocks_preview.csv") \ No newline at end of file + # Drop rows with null values + final_df.dropna(inplace=True) + + print("Writing data to file") + final_df.to_parquet(os.path.join(DATA_DIR, "stocks.parquet")) + final_df.head(200).to_csv(os.path.join(DATA_DIR, "stocks.preview.csv")) \ No newline at end of file diff --git a/WebServer/Controllers/PythonInterop.cs b/WebServer/Controllers/PythonInterop.cs new file mode 100644 index 00000000..f6f79af1 --- /dev/null +++ b/WebServer/Controllers/PythonInterop.cs @@ -0,0 +1,36 @@ +using Python.Runtime; + +namespace PythonInterop { + + public class AIModule { + + public AIModule(string PythonPath = "/usr/lib/libpython3.11.so") { + // Use the user provided python runner + Runtime.PythonDLL = PythonPath; + + string baseDir = AppDomain.CurrentDomain.BaseDirectory; + string pythonFiles = Path.Combine(baseDir, "AIPython"); + string venvRoot = Path.Combine(pythonFiles, "python"); + string venvPkgs = Path.Combine(venvRoot, "lib/python3.11/site-packages"); + + // Use our local environment for the python libraries + PythonEngine.PythonHome = venvRoot; + + // Include all the paths for python packages most importantly our venv + PythonEngine.PythonPath = $"/usr/lib/python3.11:/usr/lib/python3.11/lib-dynload:{pythonFiles}:{venvPkgs}"; + + // Initiilize python + PythonEngine.Initialize(); + } + + // This is thread blocking, runs on the main thread, and takes multiple minutes so probabaly need to run on a background thread at some point + public void PullData() { + using (Py.GIL()) { + dynamic datapuller = Py.Import("datapuller"); + dynamic result = datapuller.pull(); + } + } + + } + +} \ No newline at end of file diff --git a/WebServer/Program.cs b/WebServer/Program.cs index 24af78c9..f38f8e48 100644 --- a/WebServer/Program.cs +++ b/WebServer/Program.cs @@ -1,4 +1,5 @@ using WebServer.Components; +using PythonInterop; var builder = WebApplication.CreateBuilder(args); @@ -9,15 +10,20 @@ builder.Services.AddRazorComponents() var app = builder.Build(); // Configure the HTTP request pipeline. -if (!app.Environment.IsDevelopment()) -{ +if (!app.Environment.IsDevelopment()) { app.UseExceptionHandler("/Error", createScopeForErrors: true); // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts. app.UseHsts(); } -app.UseHttpsRedirection(); +// Load the module in globally +AIModule interopModule = new AIModule(); +// Run this for testing purposes +interopModule.PullData(); + + +app.UseHttpsRedirection(); app.UseAntiforgery(); diff --git a/WebServer/WebServer.csproj b/WebServer/WebServer.csproj index 08f689c7..bd30b0f1 100644 --- a/WebServer/WebServer.csproj +++ b/WebServer/WebServer.csproj @@ -6,4 +6,14 @@ enable + + + + + + + PreserveNewest + + +