bowong
/
modalDeploy
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

FIX AutoDL修复队列空等空指针问题

This commit is contained in:
kyj@bowong.ai 2025-04-17 17:24:54 +08:00
parent 52eeb712e0
commit d1805be4d6
3 changed files with 16 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
AutoDL/autodl_scheduling/entity/instance_pool.py
View File

@ -84,22 +84,25 @@ class InstancePool:
try: try:
before=len(self.instances) before=len(self.instances)
instance_copy = copy.deepcopy(self.instances) instance_copy = copy.deepcopy(self.instances)
flag = False flag = []
for instance in instance_copy: for instance in instance_copy:
if instance.active: if instance.active:
if (time.time() - instance.last_active_time) > self.timeout: if (time.time() - instance.last_active_time) > self.timeout:
flag = True flag.append(instance.uuid)
self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance)) self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance))
else: else:
if (time.time() - instance.last_active_time) > self.scaledown_window: if (time.time() - instance.last_active_time) > self.scaledown_window:
flag = True flag.append(instance.uuid)
self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance)) self.intro_threads.append(self.executor.submit(self.remove_instance, instance=instance))
while len(self.intro_threads) > 0: while len(self.intro_threads) > 0:
for t in self.intro_threads: for t in self.intro_threads:
t.result(timeout=self.timeout//2) t.result(timeout=self.timeout//2)
self.intro_threads.remove(t) self.intro_threads.remove(t)
after = len(self.instances) after = len(self.instances)
if flag: for instance in self.instances:
if instance.uuid in flag:
raise Exception("Instance[%s] Remove Failed" % instance.uuid)
if len(flag) > 0:
loguru.logger.info("Instance Num Before Introspecting %d After Introspecting %d" % (before, after)) loguru.logger.info("Instance Num Before Introspecting %d After Introspecting %d" % (before, after))
except Exception as e: except Exception as e:
loguru.logger.error("Fail to Introspect Instances: %s" % e) loguru.logger.error("Fail to Introspect Instances: %s" % e)

4
AutoDL/autodl_scheduling/entity/waiting_queue.py
View File

@ -11,10 +11,10 @@ class WaitingQueue:
"video_path": video_path, "video_path": video_path,
"audio_path": audio_path "audio_path": audio_path
} }
self.queue.put(data) self.queue.put(data,timeout=10)
def dequeue(self): def dequeue(self):
data = self.queue.get() data = self.queue.get(timeout=10)
return data["uid"], data["video_path"], data["audio_path"] return data["uid"], data["video_path"], data["audio_path"]
def get_size(self): def get_size(self):

10
AutoDL/autodl_scheduling/server.py
View File

@ -22,7 +22,7 @@ class Server:
self.waiting_queue = WaitingQueue() self.waiting_queue = WaitingQueue()
self.running_pool = RunningPool() self.running_pool = RunningPool()
#账号限制max_instance不能超过30 #账号限制max_instance不能超过30
self.instance_pool = InstancePool(max_instance=3) self.instance_pool = InstancePool(max_instance=2)
self.result_map = ResultMap() self.result_map = ResultMap()
self.executor = ThreadPoolExecutor(max_workers=2) self.executor = ThreadPoolExecutor(max_workers=2)
self.worker_1 = self.executor.submit(self.scaling_worker) self.worker_1 = self.executor.submit(self.scaling_worker)
@ -50,7 +50,11 @@ class Server:
try: try:
return self.result_map.get(uid) return self.result_map.get(uid)
except: except:
return {"status": "queuing", "msg":""} l = list(self.waiting_queue.queue.queue)
for i in l:
if i["uid"] == uid:
return {"status": "queuing", "msg":""}
return {"status": "not found", "msg":""}
async def get_all_result(self): async def get_all_result(self):
try: try:
@ -74,7 +78,7 @@ class Server:
# 提交任务 # 提交任务
self.instance_pool.scale_instance(self.waiting_queue.get_size()+self.running_pool.get_running_size(), disable_shrink=True) self.instance_pool.scale_instance(self.waiting_queue.get_size()+self.running_pool.get_running_size(), disable_shrink=True)
for instance in self.instance_pool.instances: for instance in self.instance_pool.instances:
if not instance.active: if not instance.active and self.waiting_queue.get_size() > 0:
# 从等待队列取出任务 # 从等待队列取出任务
uid, video_path, audio_path = self.waiting_queue.dequeue() uid, video_path, audio_path = self.waiting_queue.dequeue()
loguru.logger.info("Task[%s] Submitting to Instance[%s]" % (uid, instance.uuid)) loguru.logger.info("Task[%s] Submitting to Instance[%s]" % (uid, instance.uuid))