diff --git a/AutoDL/autodl_scheduling/server.py b/AutoDL/autodl_scheduling/server.py index f7458b4..25975a8 100644 --- a/AutoDL/autodl_scheduling/server.py +++ b/AutoDL/autodl_scheduling/server.py @@ -61,13 +61,16 @@ class Server: def introspect_instance(self): loguru.logger.info("Introspecting worker started || Scaledown Window: %ds" % self.instance_pool.scaledown_window) while True: - self.instance_pool.introspection() - time.sleep(1) + try: + self.instance_pool.introspection() + time.sleep(1) + except: + traceback.print_exc() def scaling_worker(self): loguru.logger.info("Scaling worker started") - try: - while True: + while True: + try: # 提交任务 self.instance_pool.scale_instance(self.waiting_queue.get_size()+self.running_pool.get_running_size(), disable_shrink=True) for instance in self.instance_pool.instances: @@ -104,8 +107,8 @@ class Server: instance.active = False instance.last_active_time = time.time() time.sleep(0.5) - except: - traceback.print_exc() + except: + traceback.print_exc() if __name__=="__main__":