Move away from Python Interop for better stability

WebServer/AIPython/aipredictor.py

@@ -1,4 +1,5 @@
 import os
+import sys
 import joblib
 import numpy as np
 os.environ["CUDA_VISIBLE_DEVICES"] = "-1"
@@ -7,7 +8,11 @@ import features
 import matplotlib
 matplotlib.use("Agg")
 
-def Predict(Symbol):
+def Predict():
+
+    # Get the Symbol from ARGV
+    Symbol = sys.argv[1]
+
     # Define paths (consistent with your previous script)
     SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
     DATA_DIR = os.path.join(SCRIPT_DIR, "data")
@@ -21,8 +26,6 @@ def Predict(Symbol):
         # Make the feature set
         df = features.MakeFeatures(df)
 
-    print(Symbol)
-
     # Drop our predictor
     df.drop('Target_Close', axis=1, inplace=True)
 
@@ -51,8 +54,7 @@ def Predict(Symbol):
     # 'predictions' will be a 2D array, flatten it if you want a simple list
     flat_predictions = actual_prediction.flatten().tolist()
 
-    print(f"Predicted Target_Close: {flat_predictions}")
-
+    # Set the movement indicator
     movement_indicator = 0
     if (np.mean(flat_predictions) > 0.01):
         movement_indicator = 1
@@ -61,7 +63,10 @@ def Predict(Symbol):
     else:
         movement_indicator = 0
 
-    return movement_indicator
+    # Return to C# via stdout
+    print(f"---RESULT_START---")
+    print(movement_indicator)
+    print(f"---RESULT_END---")
 
 if __name__ == "__main__":
-    Predict("AAPL")
+    Predict()

WebServer/AIPython/aitrainer.py

@@ -75,12 +75,9 @@ def TrainAI():
     test_results = dnn_model.evaluate(
         test_features, test_labels, verbose=0
     )
-    print(f"Test Results: {test_results}")
 
     # Save the model
     dnn_model.save(os.path.join(DATA_DIR, "model.keras"))
 
 if __name__ == "__main__":
     TrainAI()
-
-# Last train Predicted Target_Close: [0.0022113274317234755, 0.0021446370519697666, 0.0022628342267125845, 0.002175702480599284, 0.0021452796645462513, 0.0020838389173150063, 0.0017336219316348433, 0.002210840117186308, 0.0021144403144717216, 0.0021278387866914272, 0.0021266420371830463, 0.002261851681396365, 0.002108299173414707, 0.002121902070939541, 0.0022294146474450827]

WebServer/AIPython/currentprice.py

@@ -1,7 +1,19 @@
+import sys
 import yfinance as yf
 
-def getCurrentPrice(symbol):
+def getCurrentPrice():
+
+    # Get the Symbol from ARGV
+    symbol = sys.argv[1]
+
     ticker = yf.Ticker(symbol)
     data = ticker.history(period="1d", interval="1m")
     current_price = data['Close'].iloc[-1]
-    return current_price
+
+    # Return to C# via stdout
+    print(f"---RESULT_START---")
+    print(current_price)
+    print(f"---RESULT_END---")
+
+if __name__ == "__main__":
+    getCurrentPrice()

WebServer/AIPython/datapuller.py

@@ -15,7 +15,6 @@ def pull():
     # Scrape the data
     all_data = []
     for i, symbol in enumerate(tickers):
-        print(f"Processing: {i} of {len(tickers)}")
         df = yf.download(symbol, period="max", auto_adjust=True)
         if not df.empty:
             # Remove the ticker column
@@ -24,10 +23,11 @@ def pull():
             all_data.append(df)
 
     # Concatinate into a combined list and cache
-    print("Processing data")
     final_df = pd.concat(all_data)
 
     # Save to file
-    print("Writing data to file")
     final_df.to_parquet(os.path.join(DATA_DIR, "stocks.parquet"))
     final_df.head(200).to_csv(os.path.join(DATA_DIR, "stocks.preview.csv"))
+
+if __name__ == "__main__":
+    pull()

WebServer/Controllers/ProcessCreator.cs

@@ -0,0 +1,57 @@
+using System.Diagnostics;
+
+namespace Controllers.Processes {
+    
+    public class PyProcess {
+
+        public static (bool, string) RunPythonProcess(string PyExecutable, string PyScript, bool returns = false, string PyArgs = "") {
+            var start = new ProcessStartInfo {
+                FileName = PyExecutable,
+                Arguments = $"-u {PyScript} {PyArgs}",
+                UseShellExecute = false,
+                RedirectStandardOutput = true,
+                RedirectStandardError = true,
+                CreateNoWindow = true
+            };
+
+            using (Process? process = Process.Start(start)) {
+
+                // Try to start the process
+                if (process == null) {
+                    return (false, "Failed to start the process");
+                }
+
+                // Read the stdouts and wait for process to end
+                string result = process.StandardOutput.ReadToEnd();
+                string errors = process.StandardError.ReadToEnd();
+                process.WaitForExit();
+
+                // If the process Errored
+                if (process.ExitCode != 0) {
+                    return (false, $"Python Error : {errors}");
+                }
+
+                // If the process is supposed to return
+                if (returns) {
+                    string markerStart = "---RESULT_START---";
+                    string markerEnd = "---RESULT_END---";
+
+                    int startPos = result.IndexOf(markerStart) + markerStart.Length;
+                    int endPos = result.IndexOf(markerEnd);
+
+                    if (startPos > -1 && endPos > startPos) {
+                        string cleanResult = result.Substring(startPos, endPos - startPos).Trim();
+                        return (true, cleanResult);
+                    }
+                } else {
+                    return (true, "");
+                }
+
+                // Fail Safe
+                return (false, "Result not found");
+            }
+        }
+
+    }
+
+}

WebServer/Controllers/PythonInterop.cs

@@ -1,65 +1,55 @@
-using Python.Runtime;
+using Controllers.Processes;
 
 namespace Controllers.PythonInterop {
     
     public class AIModule {
 
-        public AIModule(string PythonPathBase = "/usr/local/", string PythonVersion = "python3.11") {
-            // Use the user provided python runner
-            Runtime.PythonDLL = PythonPathBase + $"lib/lib{PythonVersion}.so";
+        string _PyPath = "";
+        string _ExecPath = "";
 
-            // Use our local environment for the python libraries
-            PythonEngine.PythonHome = PythonPathBase;
-
-            // Include all the paths for python packages most importantly our venv
-            PythonEngine.PythonPath = $"{PythonPathBase}lib/{PythonVersion}:{PythonPathBase}lib/{PythonVersion}/lib-dynload:{PythonPathBase}lib/{PythonVersion}/site-packages:{Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AIPython")}";
-
-            // Initiilize python
-            PythonEngine.Initialize();
-
-            // Needed because C# calls the python from each connections worker thread
-            PythonEngine.BeginAllowThreads();
+        public AIModule() {
+            _PyPath = "/usr/bin/python3.11";
+            _ExecPath = $"{Path.Combine(AppDomain.CurrentDomain.BaseDirectory, "AIPython")}";
         }
 
         public void PullAI() {
-            using (Py.GIL()) {
-                dynamic datapuller = Py.Import("datapuller");
-                using (datapuller.pull()){ }
+            (bool, string) Success = PyProcess.RunPythonProcess(_PyPath, _ExecPath + "/datapuller.py");
+            if (!Success.Item1) {
+                Console.WriteLine(Success.Item2);
             }
         }
 
         public void TrainAI() {
-            using (Py.GIL()) {
-                dynamic trainer = Py.Import("ai-trainer");
-                using (trainer.TrainAI()){ }
+            (bool, string) Success = PyProcess.RunPythonProcess(_PyPath, _ExecPath + "/aitrainer.py");
+            if (!Success.Item1) {
+                Console.WriteLine(Success.Item2);
             }
         }
 
         // Return ( Error, Signal )
         public (string, int) PredictAI(string StockSymbol) {
-            try {
-                using (Py.GIL()) {
-                    dynamic predictor = Py.Import("ai-predictor");
-                    using (dynamic x = predictor.Predict(StockSymbol)) {
-                        int result = (int)x;
-                        return ("", result);
+            (bool, string) Success = PyProcess.RunPythonProcess(_PyPath, _ExecPath + "/aipredictor.py", returns: true, PyArgs: StockSymbol);
+            if (!Success.Item1) {
+                return (Success.Item2, 0);
+            } else {
+                if (int.TryParse(Success.Item2, out int parsed)) {
+                    return ("", parsed);
                 }
-                }
-            } catch (Exception ex) {
-                return (ex.ToString(), 0);
+                return ("Python returns an unknown value", 0);
             }
         }
 
         public float GetCurrentPrice(string StockSymbol) {
-            using (Py.GIL()) {
-                dynamic price = Py.Import("currentprice");
-                using (dynamic x = price.getCurrentPrice(StockSymbol)) {
-                    float CurrentPrice = (float)x;
-                    return x;
+            (bool, string) Success = PyProcess.RunPythonProcess(_PyPath, _ExecPath + "/currentprice.py", returns: true, PyArgs: StockSymbol);
+            if (!Success.Item1) {
+                return 0;
+            } else {
+                if (float.TryParse(Success.Item2, out float parsed)) {
+                    return parsed;
                 }
+                return 0;
             }
         }
 
     }
-
 }

WebServer/Program.cs

@@ -4,15 +4,29 @@ using Controllers.DataBase;
 using Controllers.Payment;
 
 // Load the module in globally and use correct path for local or docker runners
-#if DEBUG
-    AIModule interopModule = new AIModule(PythonPathBase: "/usr/", PythonVersion: "python3.11");
-#else
 AIModule interopModule = new AIModule();
-#endif
 
-if (args.Contains("Pull-Stock-Data")) {
+if (args.Contains("Retrain-AI")) {
+    // This runs once per month on the first -> set by the crontab.txt
+    interopModule.PullAI();
     interopModule.TrainAI();
+} else if (args.Contains("Perform-AI")) {
+    // This runs every hour that the stock market is open -> set by the crontab.txt
+
+    // Get all current holdings for Stocks
+    // Perform the AI Signal per stock
+    // Perform action in background for each stock
 } else {
+    // This runs when the server is started normally
+
+    // Make sure the data is ready before first run
+    string firstPullRan = (new DbDriver()).Get("FirstPull");
+    if (firstPullRan != "1") {
+        interopModule.PullAI();
+        interopModule.TrainAI();
+        (new DbDriver()).Set("FirstPull", "1");
+    }
+
     // Create the webapp
     var builder = WebApplication.CreateBuilder(args);
 

WebServer/WebServer.csproj

@@ -10,7 +10,6 @@
     <PackageReference Include="BCrypt.Net-Next" Version="4.1.0" />
     <PackageReference Include="Microsoft.Data.Sqlite" Version="10.0.3" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.4" />
-    <PackageReference Include="pythonnet" Version="3.0.5" />
   </ItemGroup>
 
   <ItemGroup>