Create a python interop service and wire it up

WebServer/AIPython/datapuller.py

@@ -1,53 +1,56 @@
+import os
 import yfinance as yf
 import pandas as pd
 
-class DataPuller:
+def pull():
 
-    @staticmethod
+    # Get the CWD for pathing due to being called from C# now
-    def pull():
+    SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
-        # Import the S&P 500 symbols
+    DATA_DIR = os.path.join(SCRIPT_DIR, "data")
-        symbols = pd.read_excel("./data/stock_symbols.xlsx")
-        symbols.columns = symbols.columns.str.strip()
-        tickers = symbols['Symbol'].tolist()
 
-        # Scrape the data
+    # Import the S&P 500 symbols
-        all_data = []
+    symbols = pd.read_excel(os.path.join(DATA_DIR, "stock_symbols.xlsx"))
-        for i, symbol in enumerate(tickers): # Try first 20
+    symbols.columns = symbols.columns.str.strip()
-            print(f"Processing: {i} of {len(tickers)}")
+    tickers = symbols['Symbol'].tolist()
-            df = yf.download(symbol, period="max", auto_adjust=True)
-            if not df.empty:
-                # Remove the ticker column
-                df.columns = df.columns.get_level_values(0)
 
-                # Make sure Date is actually a Date Object
+    # Scrape the data
-                df = df.reset_index()
+    all_data = []
-                df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d")
+    for i, symbol in enumerate(tickers): # Try first 20
-                df.set_index('Date', inplace=True)
+        print(f"Processing: {i} of {len(tickers)}")
+        df = yf.download(symbol, period="max", auto_adjust=True)
+        if not df.empty:
+            # Remove the ticker column
+            df.columns = df.columns.get_level_values(0)
             
-                # Add the Symbol column for tracking
+            # Make sure Date is actually a Date Object
-                df['Symbol'] = symbol
+            df = df.reset_index()
+            df['Date'] = pd.to_datetime(df['Date'], format="%Y-%m-%d")
+            df.set_index('Date', inplace=True)
 
-                # Add feature Spread
+            # Add the Symbol column for tracking
-                df['Spread'] = abs( df['High'] - df['Low'] )
+            df['Symbol'] = symbol
         
-                # Add feature for Returns
+            # Add feature Spread
-                df['Return'] = df['Close'].pct_change()
+            df['Spread'] = abs( df['High'] - df['Low'] )
 
-                # Add feature for volitility last 5
+            # Add feature for Returns
-                df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std())
+            df['Return'] = df['Close'].pct_change()
 
-                # Add feature for volitility last 20
+            # Add feature for volitility last 5
-                df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std())
+            df['Volatility_5'] = df['Return'].transform(lambda x: x.rolling(5).std())
 
-                all_data.append(df)
+            # Add feature for volitility last 20
+            df['Volatility_20'] = df['Return'].transform(lambda x: x.rolling(20).std())
 
-        # Concatinate into a combined list and cache
+            all_data.append(df)
-        print("Processing data")
-        final_df = pd.concat(all_data)
 
-        # Drop rows with null values
+    # Concatinate into a combined list and cache
-        final_df.dropna(inplace=True)
+    print("Processing data")
+    final_df = pd.concat(all_data)
 
-        print("Writing data to file")
+    # Drop rows with null values
-        final_df.to_parquet("./data/stocks.parquet")
+    final_df.dropna(inplace=True)
-        final_df.head(200).to_csv("./data/stocks_preview.csv")
+
+    print("Writing data to file")
+    final_df.to_parquet(os.path.join(DATA_DIR, "stocks.parquet"))
+    final_df.head(200).to_csv(os.path.join(DATA_DIR, "stocks.preview.csv"))

WebServer/Controllers/PythonInterop.cs

@@ -0,0 +1,36 @@
+using Python.Runtime;
+
+namespace PythonInterop {
+    
+    public class AIModule {
+
+        public AIModule(string PythonPath = "/usr/lib/libpython3.11.so") {
+            // Use the user provided python runner
+            Runtime.PythonDLL = PythonPath;
+
+            string baseDir = AppDomain.CurrentDomain.BaseDirectory;
+            string pythonFiles = Path.Combine(baseDir, "AIPython");
+            string venvRoot = Path.Combine(pythonFiles, "python");
+            string venvPkgs = Path.Combine(venvRoot, "lib/python3.11/site-packages");
+
+            // Use our local environment for the python libraries
+            PythonEngine.PythonHome = venvRoot;
+
+            // Include all the paths for python packages most importantly our venv
+            PythonEngine.PythonPath = $"/usr/lib/python3.11:/usr/lib/python3.11/lib-dynload:{pythonFiles}:{venvPkgs}";
+
+            // Initiilize python
+            PythonEngine.Initialize();
+        }
+
+        // This is thread blocking, runs on the main thread, and takes multiple minutes so probabaly need to run on a background thread at some point
+        public void PullData() {
+            using (Py.GIL()) {
+                dynamic datapuller = Py.Import("datapuller");
+                dynamic result = datapuller.pull();
+            }
+        }
+
+    }
+
+}

WebServer/Program.cs

@@ -1,4 +1,5 @@
 using WebServer.Components;
+using PythonInterop;
 
 var builder = WebApplication.CreateBuilder(args);
 
@@ -9,15 +10,20 @@ builder.Services.AddRazorComponents()
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.
-if (!app.Environment.IsDevelopment())
+if (!app.Environment.IsDevelopment()) {
-{
     app.UseExceptionHandler("/Error", createScopeForErrors: true);
     // The default HSTS value is 30 days. You may want to change this for production scenarios, see https://aka.ms/aspnetcore-hsts.
     app.UseHsts();
 }
 
-app.UseHttpsRedirection();
+// Load the module in globally
+AIModule interopModule = new AIModule();
 
+// Run this for testing purposes
+interopModule.PullData();
+
+
+app.UseHttpsRedirection();
 
 app.UseAntiforgery();
 

WebServer/WebServer.csproj

@@ -6,4 +6,14 @@
     <ImplicitUsings>enable</ImplicitUsings>
   </PropertyGroup>
 
+  <ItemGroup>
+    <PackageReference Include="pythonnet" Version="3.0.5" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <None Update="AIPython\**\*">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+
 </Project>