fix race conditions on lb weight update

This commit is contained in:
mantaohuang 2023-12-09 22:02:00 -05:00
parent c18fc2a76c
commit b3b7e427e7
2 changed files with 8 additions and 4 deletions

View File

@ -329,6 +329,9 @@ class OManager:
old_weights = []
while True:
await asyncio.sleep(5)
if self.lb1_pm.pid==0 or self.lb2_pm.pid==0:
# pid not acquired yet, wait a bit more
await asyncio.sleep(3)
#print("calculating weights")
new_weights = self.calc_weights()
#print("finished calc")

View File

@ -59,8 +59,8 @@ class ProcessManager:
stderr=asyncio.subprocess.PIPE)
logging.info(f"Process started, cmd: {self.cmd}")
self.proc = proc
for _ in range(3):
await asyncio.sleep(1)
for _ in range(5):
await asyncio.sleep(0.2)
got_pid = False
try:
PID = int(open(pid_fp, "r").read())
@ -106,7 +106,7 @@ class ProcessManager:
self.task.cancel()
if self.pid:
try:
os.kill(self.pid, signal.SIGINT)
os.kill(self.pid, signal.SIGTERM) # SIGINT was not enough to kill the go process...
except Exception as err:
logging.warning(f"kill failed: {err}")
self.pid = 0
@ -114,7 +114,8 @@ class ProcessManager:
def signal(self, signal):
if self.state == ProcessManagerState.RUNNING:
os.kill(self.pid, signal)
if self.pid!=0: # race condition possible when the pid has not been acquired
os.kill(self.pid, signal)
if __name__ == "__main__":