bowong
/
modalDeploy
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FIX AutoDL修复exception导致worker停止问题

This commit is contained in:
kyj@bowong.ai 2025-04-17 15:40:28 +08:00
parent 1dfab45d83
commit d0cc6f27b5
1 changed files with 9 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
AutoDL/autodl_scheduling/server.py
View File

@ -61,13 +61,16 @@ class Server:
def introspect_instance(self): def introspect_instance(self):
loguru.logger.info("Introspecting worker started || Scaledown Window: %ds" % self.instance_pool.scaledown_window) loguru.logger.info("Introspecting worker started || Scaledown Window: %ds" % self.instance_pool.scaledown_window)
while True: while True:
try:
self.instance_pool.introspection() self.instance_pool.introspection()
time.sleep(1) time.sleep(1)
except:
traceback.print_exc()
def scaling_worker(self): def scaling_worker(self):
loguru.logger.info("Scaling worker started") loguru.logger.info("Scaling worker started")
try:
while True: while True:
try:
# 提交任务 # 提交任务
self.instance_pool.scale_instance(self.waiting_queue.get_size()+self.running_pool.get_running_size(), disable_shrink=True) self.instance_pool.scale_instance(self.waiting_queue.get_size()+self.running_pool.get_running_size(), disable_shrink=True)
for instance in self.instance_pool.instances: for instance in self.instance_pool.instances: