diff --git a/WebServer/AIPython/datapuller.py b/WebServer/AIPython/datapuller.py
index 78f7120a..333ef732 100644
--- a/WebServer/AIPython/datapuller.py
+++ b/WebServer/AIPython/datapuller.py
@@ -1,53 +1,56 @@
+import os
import yfinance as yf
import pandas as pd
-class DataPuller:
+def pull():
- @staticmethod
- def pull():
- # Import the S&P 500 symbols
- symbols = pd.read_excel("./data/stock_symbols.xlsx")
- symbols.columns = symbols.columns.str.strip()
- tickers = symbols['Symbol'].tolist()
+ # Get the CWD for pathing due to being called from C# now
+ SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+ DATA_DIR = os.path.join(SCRIPT_DIR, "data")
- # Scrape the data
- all_data = []
- for i, symbol in enumerate(tickers): # Try first 20
- print(f"Processing: {i} of {len(tickers)}")
- df = yf.download(symbol, period="max", auto_adjust=True)
- if not df.empty:
- # Remove the ticker column
- df.columns = df.columns.get_level_values(0)
-
- # Make sure Date is actually a Date Object
- df = df.reset_index()
- df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d")
- df.set_index('Date', inplace=True)
+ # Import the S&P 500 symbols
+ symbols = pd.read_excel(os.path.join(DATA_DIR, "stock_symbols.xlsx"))
+ symbols.columns = symbols.columns.str.strip()
+ tickers = symbols['Symbol'].tolist()
- # Add the Symbol column for tracking
- df['Symbol'] = symbol
+ # Scrape the data
+ all_data = []
+ for i, symbol in enumerate(tickers): # Try first 20
+ print(f"Processing: {i} of {len(tickers)}")
+ df = yf.download(symbol, period="max", auto_adjust=True)
+ if not df.empty:
+ # Remove the ticker column
+ df.columns = df.columns.get_level_values(0)
- # Add feature Spread
- df['Spread'] = abs( df['High'] - df['Low'] )
+ # Make sure Date is actually a Date Object
+ df = df.reset_index()
+ df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d")
+ df.set_index('Date', inplace=True)
- # Add feature for Returns
- df['Return'] = df['Close'].pct_change()
+ # Add the Symbol column for tracking
+ df['Symbol'] = symbol
+
+ # Add feature Spread
+ df['Spread'] = abs( df['High'] - df['Low'] )
- # Add feature for volitility last 5
- df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std())
+ # Add feature for Returns
+ df['Return'] = df['Close'].pct_change()
- # Add feature for volitility last 20
- df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std())
+ # Add feature for volitility last 5
+ df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std())
- all_data.append(df)
+ # Add feature for volitility last 20
+ df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std())
- # Concatinate into a combined list and cache
- print("Processing data")
- final_df = pd.concat(all_data)
+ all_data.append(df)
- # Drop rows with null values
- final_df.dropna(inplace=True)
+ # Concatinate into a combined list and cache
+ print("Processing data")
+ final_df = pd.concat(all_data)
- print("Writing data to file")
- final_df.to_parquet("./data/stocks.parquet")
- final_df.head(200).to_csv("./data/stocks_preview.csv")
\ No newline at end of file
+ # Drop rows with null values
+ final_df.dropna(inplace=True)
+
+ print("Writing data to file")
+ final_df.to_parquet(os.path.join(DATA_DIR, "stocks.parquet"))
+ final_df.head(200).to_csv(os.path.join(DATA_DIR, "stocks.preview.csv"))
\ No newline at end of file
diff --git a/WebServer/Controllers/PythonInterop.cs b/WebServer/Controllers/PythonInterop.cs
new file mode 100644
index 00000000..f6f79af1
--- /dev/null
+++ b/WebServer/Controllers/PythonInterop.cs
@@ -0,0 +1,36 @@
+using Python.Runtime;
+
+namespace PythonInterop {
+
+ public class AIModule {
+
+ public AIModule(string PythonPath = "/usr/lib/libpython3.11.so") {
+ // Use the user provided python runner
+ Runtime.PythonDLL = PythonPath;
+
+ string baseDir = AppDomain.CurrentDomain.BaseDirectory;
+ string pythonFiles = Path.Combine(baseDir, "AIPython");
+ string venvRoot = Path.Combine(pythonFiles, "python");
+ string venvPkgs = Path.Combine(venvRoot, "lib/python3.11/site-packages");
+
+ // Use our local environment for the python libraries
+ PythonEngine.PythonHome = venvRoot;
+
+ // Include all the paths for python packages most importantly our venv
+ PythonEngine.PythonPath = $"/usr/lib/python3.11:/usr/lib/python3.11/lib-dynload:{pythonFiles}:{venvPkgs}";
+
+ // Initiilize python
+ PythonEngine.Initialize();
+ }
+
+ // This is thread blocking, runs on the main thread, and takes multiple minutes so probabaly need to run on a background thread at some point
+ public void PullData() {
+ using (Py.GIL()) {
+ dynamic datapuller = Py.Import("datapuller");
+ dynamic result = datapuller.pull();
+ }
+ }
+
+ }
+
+}
\ No newline at end of file
diff --git a/WebServer/Program.cs b/WebServer/Program.cs
index 24af78c9..f38f8e48 100644
--- a/WebServer/Program.cs
+++ b/WebServer/Program.cs
@@ -1,4 +1,5 @@
using WebServer.Components;
+using PythonInterop;
var builder = WebApplication.CreateBuilder(args);
@@ -9,15 +10,20 @@ builder.Services.AddRazorComponents()
var app = builder.Build();
// Configure the HTTP request pipeline.
-if (!app.Environment.IsDevelopment())
-{
+if (!app.Environment.IsDevelopment()) {
app.UseExceptionHandler("/Error", createScopeForErrors: true);
// The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
app.UseHsts();
}
-app.UseHttpsRedirection();
+// Load the module in globally
+AIModule interopModule = new AIModule();
+// Run this for testing purposes
+interopModule.PullData();
+
+
+app.UseHttpsRedirection();
app.UseAntiforgery();
diff --git a/WebServer/WebServer.csproj b/WebServer/WebServer.csproj
index 08f689c7..bd30b0f1 100644
--- a/WebServer/WebServer.csproj
+++ b/WebServer/WebServer.csproj
@@ -6,4 +6,14 @@
enable
+
+
+
+
+
+
+ PreserveNewest
+
+
+