From b3b7e427e78a6725280d09d4fc8e4bc89c1e3668 Mon Sep 17 00:00:00 2001
From: mantaohuang <mantao@mit.edu>
Date: Sat, 9 Dec 2023 22:02:00 -0500
Subject: [PATCH] fix race conditions on lb weight update

---
 o_manager.py       | 3 +++
 process_manager.py | 9 +++++----
 2 files changed, 8 insertions(+), 4 deletions(-)

diff --git a/o_manager.py b/o_manager.py
index ec6e15a..605f793 100644
--- a/o_manager.py
+++ b/o_manager.py
@@ -329,6 +329,9 @@ class OManager:
         old_weights = []
         while True:
             await asyncio.sleep(5)
+            if self.lb1_pm.pid==0 or self.lb2_pm.pid==0:
+                # pid not acquired yet, wait a bit more
+                await asyncio.sleep(3)
             #print("calculating weights")
             new_weights = self.calc_weights()
             #print("finished calc")
diff --git a/process_manager.py b/process_manager.py
index 7d0ff46..ecb7109 100644
--- a/process_manager.py
+++ b/process_manager.py
@@ -59,8 +59,8 @@ class ProcessManager:
                         stderr=asyncio.subprocess.PIPE)
                     logging.info(f"Process started, cmd: {self.cmd}")
                     self.proc = proc
-                    for _ in range(3):
-                        await asyncio.sleep(1)
+                    for _ in range(5):
+                        await asyncio.sleep(0.2)
                         got_pid = False
                         try:
                             PID = int(open(pid_fp, "r").read())
@@ -106,7 +106,7 @@ class ProcessManager:
         self.task.cancel()
         if self.pid:
             try:
-                os.kill(self.pid, signal.SIGINT)
+                os.kill(self.pid, signal.SIGTERM) # SIGINT was not enough to kill the go process...
             except Exception as err:
                 logging.warning(f"kill failed: {err}")
         self.pid = 0
@@ -114,7 +114,8 @@ class ProcessManager:
 
     def signal(self, signal):
         if self.state == ProcessManagerState.RUNNING:
-            os.kill(self.pid, signal)
+            if self.pid!=0: # race condition possible when the pid has not been acquired
+                os.kill(self.pid, signal)
 
 
 if __name__ == "__main__":