FIX 旧分支保留heygem server 并修复

AutoDL/autodl_scheduling/server.py

@@ -22,7 +22,7 @@ class Server:
         self.waiting_queue = WaitingQueue()
         self.running_pool = RunningPool()
         #账号限制max_instance不能超过30
-        self.instance_pool = InstancePool(max_instance=2)
+        self.instance_pool = InstancePool(max_instance=1)
         self.result_map = ResultMap()
         self.executor = ThreadPoolExecutor(max_workers=2)
         self.worker_1 = self.executor.submit(self.scaling_worker)

deploy.py

@@ -1,80 +0,0 @@
-import json
-import subprocess
-import uuid
-from pathlib import Path
-from typing import Dict
-
-import modal
-
-image = (  # build up a Modal Image to run ComfyUI, step by step
-    modal.Image.debian_slim(  # start from basic Linux with Python
-        python_version="3.10"
-    )
-    .apt_install("git")  # install git to clone ComfyUI
-    .pip_install("fastapi[standard]==0.115.4")  # install web dependencies
-    .pip_install("comfy-cli==1.3.5")  # install comfy-cli
-    .pip_install("cos-python-sdk-v5")
-    .pip_install("sqlalchemy")
-    .pip_install("ultralytics")
-    .pip_install("tencentcloud-sdk-python")
-    .pip_install("pymysql")
-    .pip_install("Pillow")
-    .pip_install("ffmpy")
-    .pip_install("opencv-python")
-    .run_commands(  # use comfy-cli to install ComfyUI and its dependencies
-        "comfy --skip-prompt install --nvidia --version 0.3.10"
-    ).add_local_file("snapshot.json","/root/snapshot.json", copy=True)
-)
-
-image = (
-    image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
-    .run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
-    .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
-    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
-    .run_commands("comfy node install https://github.com/crystian/ComfyUI-Crystools")
-    .run_commands("comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts")
-    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
-    .run_commands("comfy node install https://github.com/BennyKok/comfyui-deploy")
-    .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use")
-    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
-    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git")
-    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/CosyVoice-ComfyUI.git")
-    .run_commands("comfy node install https://github.com/jags111/efficiency-nodes-comfyui")
-    .run_commands("comfy node install https://github.com/WASasquatch/was-node-suite-comfyui")
-    .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
-    .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
-    .run_commands("comfy node install https://github.com/melMass/comfy_mtb")
-    .run_commands(
-        "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model"
-    ).run_commands(
-        "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints"
-    ).run_commands(
-        "mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT && mkdir -p /root/comfy/ComfyUI/custom_nodes/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT && rm -rf /root/comfy/ComfyUI/custom_nodes/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT /root/comfy/ComfyUI/custom_nodes/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M-SFT"
-    ).run_commands(
-        "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints"
-    ).run_commands(
-        "mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai"
-    ).run_commands(
-        "rm -rf /root/comfy/ComfyUI/models"
-    ).run_commands(
-        "apt update && apt install -y ffmpeg && ffmpeg -version"
-    ).pip_install("av")
-    # Add .run_commands(...) calls for any other custom nodes you want to download
-)
-
-image = image.add_local_file(
-    "highlight_v0.113_api.json", "/root/workflow_api.json"
-)
-
-app = modal.App(name="highlight-comfyui", image=image)
-vol = modal.Volume.from_name("comfyui-model", create_if_missing=True)
-
-@app.function(
-    allow_concurrent_inputs=10,  # required for UI startup process which runs several API calls concurrently
-    concurrency_limit=1,  # limit interactive session to 1 container
-    gpu="T4",  # good starter GPU for inference
-    volumes={"/root/comfy/ComfyUI/models": vol},  # mounts our cached models
-)
-@modal.web_server(8000, startup_timeout=60)
-def ui():
-    subprocess.Popen("comfy launch -- --listen 0.0.0.0 --port 8000", shell=True)

src/deploy.py

@@ -0,0 +1,4 @@
+import modal.cli.run
+
+if __name__ == '__main__':
+    modal.cli.run.deploy(app_ref='.\\server_cluster\\app.py', name="heygem-server", tag="initialDeploy", env="main", use_module_mode=False)

src/server_cluster/ComfyUI_Auth/web/worker.py

@@ -0,0 +1,290 @@
+# ComfyUI模板--Base Auth
+import modal
+
+comfyui_auth_image = (  # build up a Modal Image to run ComfyUI, step by step
+    modal.Image.debian_slim(  # start from basic Linux with Python
+        python_version="3.10"
+    )
+    .apt_install("git")  # install git to clone ComfyUI
+    .apt_install("gcc")
+    .apt_install("libportaudio2")
+    .pip_install("fastapi[standard]==0.115.4")  # install web dependencies
+    .apt_install("software-properties-common")
+    .apt_install("wget")
+    .run_commands("add-apt-repository -y contrib")
+    .run_commands(
+        "wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb")
+    .apt_install("cuda-toolkit")
+    .add_local_file("../whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl",
+                    copy=True)
+    .pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl")  # install modified-comfy-cli
+    .pip_install("cos-python-sdk-v5")
+    .pip_install("sqlalchemy")
+    .pip_install("ultralytics")
+    .pip_install("tencentcloud-sdk-python")
+    .pip_install("pymysql")
+    .pip_install("Pillow")
+    .pip_install("ffmpy")
+    .pip_install("opencv-python")
+    .pip_install("av")
+    .pip_install("imageio")
+    .pip_install("loguru")
+    .pip_install("conformer==0.3.2", extra_options="--no-dependencies")
+    .pip_install("einops>0.6.1", extra_options="--no-dependencies")
+    .pip_install("openai-whisper")
+    .run_commands(  # use comfy-cli to install ComfyUI and its dependencies
+        "comfy --skip-prompt install --nvidia --version 0.3.10"
+    )
+)
+
+comfyui_auth_image = (
+    comfyui_auth_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
+    .run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
+    .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
+    .run_commands("comfy node install https://github.com/crystian/ComfyUI-Crystools")
+    .run_commands("comfy node install https://github.com/pythongosssss/ComfyUI-Custom-Scripts")
+    .run_commands("comfy node install https://github.com/BennyKok/comfyui-deploy")
+    .run_commands("comfy node install https://github.com/yolain/ComfyUI-Easy-Use")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-VideoHelperSuite.git")
+    .run_commands("comfy node install https://github.com/jags111/efficiency-nodes-comfyui")
+    .run_commands("comfy node install https://github.com/WASasquatch/was-node-suite-comfyui")
+    .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
+    .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
+    .run_commands("comfy node install https://github.com/melMass/comfy_mtb")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
+    .run_commands(
+        "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model"
+    ).run_commands(
+        "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints"
+    ).run_commands(
+        "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints"
+    ).run_commands(
+        "mkdir -p /root/comfy/ComfyUI/models/stabilityai && ln -s /root/comfy/ComfyUI/models/stabilityai /root/comfy/ComfyUI/stabilityai"
+    ).run_commands(
+        "mkdir -p /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M && mkdir -p /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && rm -rf /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M && ln -s /root/comfy/ComfyUI/models/CosyVoice-ComfyUI/pretrained_models/CosyVoice-300M /root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/CosyVoice-300M"
+    ).run_commands(
+        "rm -rf /root/comfy/ComfyUI/models"
+    ).run_commands(
+        "apt update && apt install -y ffmpeg && ffmpeg -version"
+    ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",
+                     copy=True
+                     ).add_local_file("../config/config.py",
+                                      "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py",
+                                      copy=True
+                                      ).workdir("/root/comfy").add_local_python_source("ComfyUI_Auth")
+)
+
+comfyui_auth_app = modal.App(name="ComfyUI-Auth", image=comfyui_auth_image, include_source=False)
+comfyui_auth_app.set_description("ComfyUI Auth Server")
+
+with comfyui_auth_image.imports():
+    import json
+    import os
+    import shutil
+    import subprocess
+    import uuid
+    from typing import Dict
+    import loguru
+    from fastapi import Depends, HTTPException, status
+    from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
+vol = modal.Volume.from_name("comfyui-model", create_if_missing=True)
+bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False)
+bucket_input = str(bucket_dict.get("INPUT"))
+bucket_output = str(bucket_dict.get("OUTPUT"))
+secret = modal.Secret.from_name("aws-s3-secret")
+output_dir = "/root/comfy/ComfyUI/output"
+auth_scheme = HTTPBearer()
+
+
+@comfyui_auth_app.cls(
+    max_containers=200,
+    min_containers=0,
+    buffer_containers=0,
+    scaledown_window=120,
+    timeout=1200,
+    gpu=["L4", "T4"],
+    cpu=(2, 16),
+    memory=(20480, 131072),
+    enable_memory_snapshot=False,
+    secrets=[secret, modal.Secret.from_name("web_auth_token")],
+    volumes={
+        "/root/comfy/ComfyUI/models": vol,
+        "/root/comfy/ComfyUI/input_s3": modal.CloudBucketMount(
+            bucket_name=bucket_input,
+            # bucket_endpoint_url="https://s3.%s.amazonaws.com" % os.environ["AWS_REGION"],
+            secret=secret,
+            key_prefix="/"
+        ),
+        "/root/comfy/ComfyUI/output_s3": modal.CloudBucketMount(
+            bucket_name=bucket_output,
+            # bucket_endpoint_url="https://s3.%s.amazonaws.com" % os.environ["AWS_REGION"],
+            secret=secret,
+            key_prefix="/"
+        ),
+    },
+)
+@modal.concurrent(max_inputs=1)
+class ComfyUIAuth:
+    @modal.enter()
+    def launch_comfy_background(self):
+        self.session_id = str(uuid.uuid4())
+        cmd = "echo client_uuid: {} && comfy launch --background".format(
+            self.session_id, self.session_id, self.session_id)
+        subprocess.run(cmd, shell=True, check=True)
+
+    @modal.method()
+    def infer(self, workflow_json: str = ""):
+        self.poll_server_health()
+        self.prompt_uuid = str(uuid.uuid4())
+        workflow = json.loads(workflow_json)
+        print("Workflow JSON:")
+        print(json.dumps(workflow, indent=4, ensure_ascii=False))
+        for node in workflow.values():
+            if node.get("class_type") == "ComfyUIDeployExternalText":
+                if node.get("inputs").get("input_id") == "file_path":
+                    file_to_move = node.get("inputs").get("default_value")
+                    if file_to_move:
+                        os.makedirs(os.path.dirname(file_to_move.replace("input_s3", "input")), exist_ok=True)
+                        try:
+                            shutil.copy(file_to_move, file_to_move.replace("input_s3", "input"))
+                        except:
+                            try:
+                                print("Try download file from S3 manually")
+                                # S3 Fallback
+                                import boto3
+                                import yaml
+                                with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",
+                                          encoding="utf-8", mode="r+") as config:
+                                    yaml_config = yaml.load(config, Loader=yaml.FullLoader)
+                                awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
+                                                       aws_secret_access_key=yaml_config["aws_access_key"])
+                                awss3.meta.client.download_file(bucket_input, file_to_move.split("input_s3/")[1],
+                                                                file_to_move.replace("input_s3", "input"))
+                            except:
+                                raise Exception("Failed to download file from S3 manually")
+                        node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3",
+                                                                                                  "input")
+        with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as fi:
+            fi.write(json.dumps(workflow, ensure_ascii=False))
+        cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose"
+        # looks up the name of the output image file based on the workflow
+        self.file_prefix = [
+            node.get("inputs")
+            for node in workflow.values()
+            if node.get("class_type") == "VHS_VideoCombine"
+        ][0]["filename_prefix"]
+
+        subprocess.run(cmd, shell=True, check=True, timeout=1195)
+
+        # returns the image as bytes
+        file_list = os.listdir(output_dir)
+        # 获取按照文件时间创建排序的列表，默认是按时间升序
+        new_file_list = sorted(file_list, key=lambda file: os.path.getctime(os.path.join(output_dir, file)),
+                               reverse=True)
+        for f in new_file_list:
+            if f.startswith(self.file_prefix):
+                os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)),
+                            exist_ok=True)
+                try:
+                    shutil.copy(os.path.join(output_dir, f),
+                                os.path.join(output_dir.replace("output", "output_s3"), f))
+                except:
+                    try:
+                        print("Try move file to S3 manually")
+                        # S3 Fallback
+                        import boto3
+                        import yaml
+                        with open("/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml",
+                                  encoding="utf-8", mode="r+") as config:
+                            yaml_config = yaml.load(config, Loader=yaml.FullLoader)
+                        awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
+                                               aws_secret_access_key=yaml_config["aws_access_key"])
+                        awss3.meta.client.upload_file(os.path.join(output_dir, f), bucket_output, f)
+                    except:
+                        raise Exception("Failed to move file to S3 manually")
+                return f
+
+    @modal.fastapi_endpoint(method="POST")
+    def api(self, item: Dict, token: HTTPAuthorizationCredentials = Depends(auth_scheme)):
+        if token.credentials != os.environ["AUTH_TOKEN"]:
+            raise HTTPException(
+                status_code=status.HTTP_401_UNAUTHORIZED,
+                detail="Incorrect bearer token",
+                headers={"WWW-Authenticate": "Bearer"},
+            )
+        try:
+            # save this updated workflow to a new file
+            new_workflow_file = item["prompt"]
+            # run inference on the currently running container
+            fname = self.infer.local(new_workflow_file)
+            if fname is None:
+                raise RuntimeError("Output File not found")
+            j = {"status": "success", "file_name": fname}
+            loguru.logger.success(j)
+            return j
+        except Exception as e:
+            j = {"status": "fail", "msg": str(e)}
+            loguru.logger.error(j)
+            return j
+        finally:
+            print("Purge Garbage By Restart Comfy")
+            cmd = "comfy stop"
+            try:
+                subprocess.run(cmd, shell=True, check=True)
+            except:
+                pass
+            print("Summary Logs")
+            try:
+                with open("/root/comfy/ComfyUI/user/comfyui.log", "r", encoding="utf-8") as f:
+                    log_text = f"\n-----------{self.file_prefix}------------\n" + f.read() + "\n"
+                    if not os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"):
+                        os.makedirs(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}", exist_ok=True)
+                    if os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt"):
+                        with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r",
+                                  encoding="utf-8") as f:
+                            log_text = f.read() + log_text
+                    with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "w",
+                              encoding="utf-8") as f:
+                        f.write(log_text)
+                if os.path.exists(f"/root/comfy/ComfyUI/user/ffmpeg.txt"):
+                    shutil.copy("/root/comfy/ComfyUI/user/ffmpeg.txt",
+                                f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/ffmpeg.txt")
+            except Exception as e:
+                print(f"Summary Logs Failed: {e}")
+            cmd = "comfy launch --background"
+            try:
+                subprocess.run(cmd, shell=True, check=True)
+            except:
+                pass
+
+    def poll_server_health(self):
+        import socket
+        import urllib
+
+        try:
+            # dummy request to check if the server is healthy
+            req = urllib.request.Request("http://127.0.0.1:8188/system_stats")
+            urllib.request.urlopen(req, timeout=5)
+            print("ComfyUI server is healthy")
+        except (socket.timeout, urllib.error.URLError) as e:
+            # if no response in 5 seconds, stop the container; Modal will schedule queued inputs on a new container
+            print(f"Server health check failed: {str(e)} restarting ComfyUI...")
+            try:
+                cmd = "comfy stop"
+                try:
+                    subprocess.run(cmd, shell=True, check=True)
+                except:
+                    pass
+                cmd = "comfy launch --background"
+                try:
+                    subprocess.run(cmd, shell=True, check=True)
+                except:
+                    raise Exception("Failed to launch ComfyUI")
+            except:
+                modal.experimental.stop_fetching_inputs()
+                raise Exception("ComfyUI server is not healthy, restart failed, stopping container")

src/server_cluster/ComfyUI_Auth_HeyGem/web/worker.py

@@ -1,17 +1,7 @@
 # ComfyUI模板--Base Auth Heygem
-import json
-import os
-import shutil
-import subprocess
-import uuid
-from typing import Dict
-
-import loguru
 import modal
-from fastapi import Depends, HTTPException, status
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 
-image = (  # build up a Modal Image to run ComfyUI, step by step
+comfyui_auth_heygem_image = (  # build up a Modal Image to run ComfyUI, step by step
     modal.Image.debian_slim(  # start from basic Linux with Python
         python_version="3.10"
     )
@@ -39,8 +29,8 @@ image = (  # build up a Modal Image to run ComfyUI, step by step
     )
 )
 
-image = (
-    image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
+comfyui_auth_heygem_image = (
+    comfyui_auth_heygem_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
     .run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
     .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
     .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
@@ -54,10 +44,10 @@ image = (
     .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
     .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
     .run_commands("comfy node install https://github.com/melMass/comfy_mtb")
-    .run_commands("echo 3 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
-    .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
-    .run_commands("echo 04111 && comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
-    .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
     .run_commands(
         "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model"
     ).run_commands(
@@ -72,46 +62,52 @@ image = (
         "rm -rf /root/comfy/ComfyUI/models"
     ).run_commands(
         "apt update && apt install -y ffmpeg && ffmpeg -version"
-    ).add_local_file("config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True
-    ).add_local_file("config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True
+    ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True
+    ).add_local_file("../config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True
     ) # 添加Python3.8 HeyGem
     .run_commands("apt update && apt install -y curl build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev lzma liblzma-dev tk-dev libffi-dev")
     .run_commands("curl -O https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz&&tar -xf Python-3.8.12.tar.xz")
     .run_commands("cd Python-3.8.12 && ./configure --enable-optimizations && make -j 10 && make altinstall")
-    .add_local_file("heygem-1.0-py3-none-any.whl","/root/comfy/heygem-1.0-py3-none-any.whl", copy=True)
+    .add_local_file("../whl/heygem-1.0-py3-none-any.whl", "/root/heygem-1.0-py3-none-any.whl", copy=True)
     .shell(["/bin/bash", "-c"])
-    .run_commands("python3.8 -m pip install /root/comfy/heygem-1.0-py3-none-any.whl")
+    .run_commands("python3.8 -m pip install /root/heygem-1.0-py3-none-any.whl")
     .env({"LD_LIBRARY_PATH":"/usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib"})
     .run_commands("ln -s /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so.11.2 /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so")
     .run_commands("python3.8 -m pip install https://github.com/pydata/numexpr/releases/download/v2.8.6/numexpr-2.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
     .add_local_file("heygem.py", "/root/comfy/heygem.py", copy=True)
-    .workdir("/root/comfy")
-
-    # Add .run_commands(...) calls for any other custom nodes you want to download
+    .workdir("/root/comfy").add_local_python_source("ComfyUI_Auth_HeyGem")
 )
 
-app = modal.App(name="highlight-comfyui-s3", image=image)
+comfyui_auth_heygem_app = modal.App(name="ComfyUI-Auth-HeyGem", image=comfyui_auth_heygem_image, include_source=False)
+comfyui_auth_heygem_app.set_description("ComfyUI Auth HeyGem Server")
+
+with comfyui_auth_heygem_image.imports():
+    import json
+    import os
+    import shutil
+    import subprocess
+    import uuid
+    from typing import Dict
+    import loguru
+    from fastapi import Depends, HTTPException, status
+    from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
 vol = modal.Volume.from_name("comfyui-model", create_if_missing=True)
 bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False)
 bucket_input = str(bucket_dict.get("INPUT"))
 bucket_output = str(bucket_dict.get("OUTPUT"))
 secret = modal.Secret.from_name("aws-s3-secret")
-# completed workflows write output images to this directory
 output_dir = "/root/comfy/ComfyUI/output"
 auth_scheme = HTTPBearer()
 
-
-@app.cls(
-    allow_concurrent_inputs=1,  # allow 10 concurrent API calls
+@comfyui_auth_heygem_app.cls(
     max_containers=200,
     min_containers=0,
     buffer_containers=0,
     scaledown_window=120,
-    # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times
     timeout=1200,
     gpu=["L4", "T4"],
     cpu=(2,16),
-    # memory=(32768, 32768), # (内存预留量, 内存使用上限)
     memory=(20480,81920),
     enable_memory_snapshot=False,
     secrets=[secret, modal.Secret.from_name("web_auth_token")],
@@ -131,10 +127,10 @@ auth_scheme = HTTPBearer()
         ),
     },
 )
-class ComfyUI:
+@modal.concurrent(max_inputs=1)
+class ComfyUIAuthHeyGem:
     @modal.enter()
     def launch_comfy_background(self):
-        # starts the ComfyUI server in the background exactly once when the first input is received
         self.session_id = str(uuid.uuid4())
         cmd = ("echo client_uuid: {}"
                " && rm -rf /root/comfy/ComfyUI/user"
@@ -150,7 +146,6 @@ class ComfyUI:
     def infer(self, workflow_json: str = ""):
         self.poll_server_health()
         self.prompt_uuid = str(uuid.uuid4())
-        # runs the comfy run --workflow command as a subprocess
         workflow = json.loads(workflow_json)
         print("Workflow JSON:")
         print(json.dumps(workflow, indent=4, ensure_ascii=False))
@@ -249,8 +244,6 @@ class ComfyUI:
             raise Exception("Workflow does not contain VHS_VideoCombine/HeygemF2F node, cannot find output file")
 
 
-
-
     @modal.fastapi_endpoint(method="POST")
     def api(self, item: Dict, token: HTTPAuthorizationCredentials = Depends(auth_scheme)):
         if token.credentials != os.environ["AUTH_TOKEN"]:
@@ -313,8 +306,3 @@ class ComfyUI:
                 modal.experimental.stop_fetching_inputs()
                 raise Exception("ComfyUI server is not healthy, restart failed, stopping container")
 
-    # @modal.web_endpoint(method="POST", label="tk")
-    # def tk_api(self, item: Dict):
-    #     pass
-    #
-

src/server_cluster/ComfyUI_Auth_LatentSync1_5/web/worker.py

@@ -1,17 +1,7 @@
 # ComfyUI模板--Base Auth
-import json
-import os
-import shutil
-import subprocess
-import uuid
-from typing import Dict
-
-import loguru
 import modal
-from fastapi import Depends, HTTPException, status
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
 
-image = (  # build up a Modal Image to run ComfyUI, step by step
+comfyui_auth_latentsync_1_5_image = (  # build up a Modal Image to run ComfyUI, step by step
     modal.Image.debian_slim(  # start from basic Linux with Python
         python_version="3.10"
     )
@@ -19,7 +9,13 @@ image = (  # build up a Modal Image to run ComfyUI, step by step
     .apt_install("gcc")
     .apt_install("libportaudio2")
     .pip_install("fastapi[standard]==0.115.4")  # install web dependencies
-    .pip_install("comfy-cli==1.3.5")  # install comfy-cli
+    .apt_install("software-properties-common")
+    .apt_install("wget")
+    .run_commands("add-apt-repository -y contrib")
+    .run_commands("wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb")
+    .apt_install("cuda-toolkit")
+    .add_local_file("../whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True)
+    .pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl")  # install modified-comfy-cli
     .pip_install("cos-python-sdk-v5")
     .pip_install("sqlalchemy")
     .pip_install("ultralytics")
@@ -39,8 +35,8 @@ image = (  # build up a Modal Image to run ComfyUI, step by step
     )
 )
 
-image = (
-    image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
+comfyui_auth_latentsync_1_5_image = (
+    comfyui_auth_latentsync_1_5_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
     .run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
     .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
     .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
@@ -54,14 +50,14 @@ image = (
     .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
     .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
     .run_commands("comfy node install https://github.com/melMass/comfy_mtb")
-    .run_commands("echo 3 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
     .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
-    .run_commands("echo 04111 && comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
-    .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/Comfy-LatentSync-Next-Node.git")
     .run_commands(
         "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model"
     ).run_commands(
-        "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/ComfyUI-LatentSync-Node/checkpoints"
+        "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints && rm -rf /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints && ln -s /root/comfy/ComfyUI/models/ComfyUI-LatentSync-Next-Node/checkpoints /root/comfy/ComfyUI/custom_nodes/Comfy-LatentSync-Next-Node/checkpoints"
     ).run_commands(
         "mkdir -p /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints && mkdir -p /root/.cache/torch/hub/checkpoints && rm -rf /root/.cache/torch/hub/checkpoints && ln -s /root/comfy/ComfyUI/models/.cache/torch/hub/checkpoints /root/.cache/torch/hub/checkpoints"
     ).run_commands(
@@ -72,35 +68,43 @@ image = (
         "rm -rf /root/comfy/ComfyUI/models"
     ).run_commands(
         "apt update && apt install -y ffmpeg && ffmpeg -version"
-    ).add_local_file("config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True
-    ).add_local_file("config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True
-    ).workdir("/root/comfy")
-
-    # Add .run_commands(...) calls for any other custom nodes you want to download
+    ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True
+    ).add_local_file("../config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True
+    ).workdir("/root/comfy").add_local_python_source("ComfyUI_Auth_LatentSync1_5")
 )
 
-app = modal.App(name="highlight-comfyui-s3", image=image)
+comfyui_auth_latentsync_1_5_app = modal.App(name="ComfyUI-Auth-LatentSync1_5", image=comfyui_auth_latentsync_1_5_image, include_source=False)
+comfyui_auth_latentsync_1_5_app.set_description("ComfyUI Auth LatentSync1_5 Server")
+
+
+with comfyui_auth_latentsync_1_5_image.imports():
+    import json
+    import os
+    import shutil
+    import subprocess
+    import time
+    import uuid
+    from typing import Dict
+    import loguru
+    from fastapi import Depends, HTTPException, status
+    from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
 vol = modal.Volume.from_name("comfyui-model", create_if_missing=True)
 bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False)
 bucket_input = str(bucket_dict.get("INPUT"))
 bucket_output = str(bucket_dict.get("OUTPUT"))
 secret = modal.Secret.from_name("aws-s3-secret")
-# completed workflows write output images to this directory
 output_dir = "/root/comfy/ComfyUI/output"
 auth_scheme = HTTPBearer()
 
-
-@app.cls(
-    allow_concurrent_inputs=1,  # allow 10 concurrent API calls
+@comfyui_auth_latentsync_1_5_app.cls(
     max_containers=200,
     min_containers=0,
     buffer_containers=0,
     scaledown_window=120,
-    # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times
     timeout=900,
-    gpu=["L4", "T4"],
+    gpu=["L4"],
     cpu=(2,16),
-    # memory=(32768, 32768), # (内存预留量, 内存使用上限)
     memory=(32768,131072),
     enable_memory_snapshot=False,
     secrets=[secret, modal.Secret.from_name("web_auth_token")],
@@ -120,19 +124,18 @@ auth_scheme = HTTPBearer()
         ),
     },
 )
-class ComfyUI:
+@modal.concurrent(max_inputs=1)
+class ComfyUIAuthLatentSync15:
     @modal.enter()
     def launch_comfy_background(self):
-        # starts the ComfyUI server in the background exactly once when the first input is received
         self.session_id = str(uuid.uuid4())
-        cmd = "echo client_uuid: {}&&rm -rf /root/comfy/ComfyUI/user&&mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}&&ln -s /root/comfy/ComfyUI/output_s3/logs/{} /root/comfy/ComfyUI/user && comfy launch --background".format(self.session_id,self.session_id, self.session_id)
+        cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id,self.session_id, self.session_id)
         subprocess.run(cmd, shell=True, check=True)
 
     @modal.method()
     def infer(self, workflow_json: str = ""):
         self.poll_server_health()
         self.prompt_uuid = str(uuid.uuid4())
-        # runs the comfy run --workflow command as a subprocess
         workflow = json.loads(workflow_json)
         print("Workflow JSON:")
         print(json.dumps(workflow, indent=4, ensure_ascii=False))
@@ -161,15 +164,16 @@ class ComfyUI:
         with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as fi:
             fi.write(json.dumps(workflow, ensure_ascii=False))
         cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 890 --verbose"
-        subprocess.run(cmd, shell=True, check=True, timeout=895)
 
         # looks up the name of the output image file based on the workflow
-        file_prefix = [
+        self.file_prefix = [
             node.get("inputs")
             for node in workflow.values()
             if node.get("class_type") == "VHS_VideoCombine"
         ][0]["filename_prefix"]
 
+        subprocess.run(cmd, shell=True, check=True, timeout=895)
+
         # returns the image as bytes
         file_list = os.listdir(output_dir)
         # 获取按照文件时间创建排序的列表，默认是按时间升序
@@ -177,7 +181,7 @@ class ComfyUI:
                                reverse=True)
         # print("file_list", new_file_list)
         for f in new_file_list:
-            if f.startswith(file_prefix):
+            if f.startswith(self.file_prefix):
                 os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)), exist_ok=True)
                 try:
                     shutil.copy(os.path.join(output_dir, f), os.path.join(output_dir.replace("output", "output_s3"), f))
@@ -223,8 +227,24 @@ class ComfyUI:
             cmd = "comfy stop"
             try:
                 subprocess.run(cmd, shell=True, check=True)
+                time.sleep(1)
             except:
                 pass
+            print("Summary Logs")
+            try:
+                with open("/root/comfy/ComfyUI/user/comfyui.log", "r", encoding="utf-8") as f:
+                    log_text = f"\n-----------{self.file_prefix}------------\n"+f.read()+"\n"
+                    if not os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"):
+                        os.makedirs(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}", exist_ok=True)
+                    if os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt"):
+                        with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r", encoding="utf-8") as f:
+                            log_text = f.read() + log_text
+                    with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "w", encoding="utf-8") as f:
+                        f.write(log_text)
+                if os.path.exists(f"/root/comfy/ComfyUI/user/ffmpeg.txt"):
+                    shutil.copy("/root/comfy/ComfyUI/user/ffmpeg.txt", f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/ffmpeg.txt")
+            except Exception as e:
+                print(f"Summary Logs Failed: {e}")
             cmd = "comfy launch --background"
             try:
                 subprocess.run(cmd, shell=True, check=True)
@@ -257,5 +277,3 @@ class ComfyUI:
             except:
                 modal.experimental.stop_fetching_inputs()
                 raise Exception("ComfyUI server is not healthy, restart failed, stopping container")
-
-

src/server_cluster/ComfyUI_Base/web/worker.py

@@ -1,15 +1,7 @@
 # ComfyUI模板--Base 停止维护！！
-import json
-import os
-import shutil
-import subprocess
-import uuid
-from typing import Dict
-
-import loguru
 import modal
 
-image = (  # build up a Modal Image to run ComfyUI, step by step
+comfyui_base_image = (  # build up a Modal Image to run ComfyUI, step by step
     modal.Image.debian_slim(  # start from basic Linux with Python
         python_version="3.10"
     )
@@ -17,7 +9,13 @@ image = (  # build up a Modal Image to run ComfyUI, step by step
     .apt_install("gcc")
     .apt_install("libportaudio2")
     .pip_install("fastapi[standard]==0.115.4")  # install web dependencies
-    .pip_install("comfy-cli==1.3.5")  # install comfy-cli
+    .apt_install("software-properties-common")
+    .apt_install("wget")
+    .run_commands("add-apt-repository -y contrib")
+    .run_commands("wget https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/cuda-keyring_1.1-1_all.deb&&dpkg -i cuda-keyring_1.1-1_all.deb")
+    .apt_install("cuda-toolkit")
+    .add_local_file("../whl/comfy_cli-0.0.0-py3-none-any.whl", "/root/comfy_cli-0.0.0-py3-none-any.whl", copy=True)
+    .pip_install("/root/comfy_cli-0.0.0-py3-none-any.whl")  # install modified-comfy-cli
     .pip_install("cos-python-sdk-v5")
     .pip_install("sqlalchemy")
     .pip_install("ultralytics")
@@ -37,8 +35,8 @@ image = (  # build up a Modal Image to run ComfyUI, step by step
     )
 )
 
-image = (
-    image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
+comfyui_base_image = (
+    comfyui_base_image.run_commands("comfy node install https://github.com/M1kep/ComfyLiterals")
     .run_commands("comfy node install https://github.com/evanspearman/ComfyMath")
     .run_commands("comfy node install https://github.com/Kosinkadink/ComfyUI-AnimateDiff-Evolved")
     .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-Bowong.git")
@@ -52,10 +50,10 @@ image = (
     .run_commands("comfy node install https://github.com/rgthree/rgthree-comfy")
     .run_commands("comfy node install https://github.com/cubiq/ComfyUI_essentials")
     .run_commands("comfy node install https://github.com/melMass/comfy_mtb")
-    .run_commands("echo 3 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI_SparkTTS.git")
     .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-CustomNode.git")
-    .run_commands("echo 04111 && comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
-    .run_commands("echo 4 && comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/cosyvoice_comfyui.git")
+    .run_commands("comfy node install https://e.coding.net/g-ldyi2063/dev/ComfyUI-LatentSync-Node.git")
     .run_commands(
         "mkdir -p /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model && rm -rf /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model && ln -s /root/comfy/ComfyUI/models/ComfyUI-CustomNode/model /root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/model"
     ).run_commands(
@@ -70,35 +68,39 @@ image = (
         "rm -rf /root/comfy/ComfyUI/models"
     ).run_commands(
         "apt update && apt install -y ffmpeg && ffmpeg -version"
-    ).add_local_file("config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True
-    ).add_local_file("config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True
-    ).workdir("/root/comfy")
-
-    # Add .run_commands(...) calls for any other custom nodes you want to download
+    ).add_local_file("../config/config.yaml", "/root/comfy/ComfyUI/custom_nodes/ComfyUI-CustomNode/config.yaml", copy=True
+    ).add_local_file("../config/config.py", "/root/comfy/ComfyUI/custom_nodes/cosyvoice_comfyui/pretrained_models/tools/config.py", copy=True
+    ).workdir("/root/comfy").add_local_python_source("ComfyUI_Base")
 )
 
-app = modal.App(name="highlight-comfyui-s3", image=image)
+comfyui_base_app = modal.App(name="ComfyUI-Base", image=comfyui_base_image, include_source=False)
+comfyui_base_app.set_description("ComfyUI Base Server")
+
+with comfyui_base_image.imports():
+    import json
+    import os
+    import shutil
+    import subprocess
+    import uuid
+    from typing import Dict
+    import loguru
+
 vol = modal.Volume.from_name("comfyui-model", create_if_missing=True)
 bucket_dict = modal.Dict.from_name("aws_s3_bucket", create_if_missing=False)
 bucket_input = str(bucket_dict.get("INPUT"))
 bucket_output = str(bucket_dict.get("OUTPUT"))
 secret = modal.Secret.from_name("aws-s3-secret")
-# completed workflows write output images to this directory
 output_dir = "/root/comfy/ComfyUI/output"
 
-
-@app.cls(
-    allow_concurrent_inputs=1,  # allow 10 concurrent API calls
+@comfyui_base_app.cls(
     max_containers=200,
     min_containers=0,
     buffer_containers=0,
     scaledown_window=120,
-    # 5 minute container keep alive after it processes an input; increasing this value is a great way to reduce ComfyUI cold start times
-    timeout=900,
+    timeout=1200,
     gpu=["L4", "T4"],
     cpu=(2,16),
-    # memory=(32768, 32768), # (内存预留量, 内存使用上限)
-    memory=(20480,40960),
+    memory=(20480,65536),
     enable_memory_snapshot=False,
     secrets=[secret],
     volumes={
@@ -117,19 +119,18 @@ output_dir = "/root/comfy/ComfyUI/output"
         ),
     },
 )
-class ComfyUI:
+@modal.concurrent(max_inputs=1)
+class ComfyUIBase:
     @modal.enter()
     def launch_comfy_background(self):
-        # starts the ComfyUI server in the background exactly once when the first input is received
         self.session_id = str(uuid.uuid4())
-        cmd = "echo client_uuid: {}&&rm -rf /root/comfy/ComfyUI/user&&mkdir -p /root/comfy/ComfyUI/output_s3/logs/{}&&ln -s /root/comfy/ComfyUI/output_s3/logs/{} /root/comfy/ComfyUI/user && comfy launch --background".format(self.session_id,self.session_id, self.session_id)
+        cmd = "echo client_uuid: {} && comfy launch --background".format(self.session_id,self.session_id, self.session_id)
         subprocess.run(cmd, shell=True, check=True)
 
     @modal.method()
     def infer(self, workflow_json: str = ""):
         self.poll_server_health()
         self.prompt_uuid = str(uuid.uuid4())
-        # runs the comfy run --workflow command as a subprocess
         workflow = json.loads(workflow_json)
         print("Workflow JSON:")
         print(json.dumps(workflow, indent=4, ensure_ascii=False))
@@ -143,7 +144,7 @@ class ComfyUI:
                             shutil.copy(file_to_move, file_to_move.replace("input_s3", "input"))
                         except:
                             try:
-                                print("Try download file to S3 manually")
+                                print("Try download file from S3 manually")
                                 # S3 Fallback
                                 import boto3
                                 import yaml
@@ -159,16 +160,17 @@ class ComfyUI:
                         node["inputs"]["default_value"] = node["inputs"]["default_value"].replace("input_s3", "input")
         with open(f"/root/{self.prompt_uuid}.json", "w", encoding="utf-8") as f:
             f.write(json.dumps(workflow, ensure_ascii=False))
-        cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 890 --verbose"
-        subprocess.run(cmd, shell=True, check=True, timeout=895)
-
+        cmd = f"comfy run --workflow /root/{self.prompt_uuid}.json --wait --timeout 1190 --verbose"
         # looks up the name of the output image file based on the workflow
-        file_prefix = [
+        self.file_prefix = [
             node.get("inputs")
             for node in workflow.values()
             if node.get("class_type") == "VHS_VideoCombine"
         ][0]["filename_prefix"]
 
+        subprocess.run(cmd, shell=True, check=True, timeout=1195)
+
+
         # returns the image as bytes
         file_list = os.listdir(output_dir)
         # 获取按照文件时间创建排序的列表，默认是按时间升序
@@ -176,7 +178,7 @@ class ComfyUI:
                                reverse=True)
         # print("file_list", new_file_list)
         for f in new_file_list:
-            if f.startswith(file_prefix):
+            if f.startswith(self.file_prefix):
                 os.makedirs(os.path.dirname(os.path.join(output_dir.replace("output", "output_s3"), f)), exist_ok=True)
                 try:
                     shutil.copy(os.path.join(output_dir, f), os.path.join(output_dir.replace("output", "output_s3"), f))
@@ -217,6 +219,21 @@ class ComfyUI:
                 subprocess.run(cmd, shell=True, check=True)
             except:
                 pass
+            print("Summary Logs")
+            try:
+                with open("/root/comfy/ComfyUI/user/comfyui.log", "r", encoding="utf-8") as f:
+                    log_text = f"\n-----------{self.file_prefix}------------\n"+f.read()+"\n"
+                    if not os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}"):
+                        os.makedirs(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}", exist_ok=True)
+                    if os.path.exists(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt"):
+                        with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "r", encoding="utf-8") as f:
+                            log_text = f.read() + log_text
+                    with open(f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/full.txt", "w", encoding="utf-8") as f:
+                        f.write(log_text)
+                if os.path.exists(f"/root/comfy/ComfyUI/user/ffmpeg.txt"):
+                    shutil.copy("/root/comfy/ComfyUI/user/ffmpeg.txt", f"/root/comfy/ComfyUI/output_s3/logs/{self.session_id}/ffmpeg.txt")
+            except Exception as e:
+                print(f"Summary Logs Failed: {e}")
             cmd = "comfy launch --background"
             try:
                 subprocess.run(cmd, shell=True, check=True)
@@ -250,9 +267,3 @@ class ComfyUI:
             except:
                 modal.experimental.stop_fetching_inputs()
                 raise Exception("ComfyUI server is not healthy, restart failed, stopping container")
-
-    # @modal.web_endpoint(method="POST", label="tk")
-    # def tk_api(self, item: Dict):
-    #     pass
-    #
-

src/server_cluster/HeyGem_Base/web/worker.py

@@ -1,21 +1,7 @@
 # HeyGem模板
-import json
-import os
-import shutil
-import socket
-import subprocess
-import time
-import traceback
-import uuid
-from typing import Any, Optional, Union
-
-import httpx
-import loguru
 import modal
-import requests
-from fastapi import Depends, HTTPException, status, UploadFile
-from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
-image = (
+
+heygem_base_image = (
     modal.Image.debian_slim(  # start from basic Linux with Python
         python_version="3.10"
     ).apt_install("git")
@@ -29,47 +15,69 @@ image = (
     .run_commands("apt update && apt install -y curl build-essential libssl-dev zlib1g-dev libncurses5-dev libncursesw5-dev libreadline-dev libsqlite3-dev libgdbm-dev libdb5.3-dev libbz2-dev libexpat1-dev lzma liblzma-dev tk-dev libffi-dev")
     .run_commands("curl -O https://www.python.org/ftp/python/3.8.12/Python-3.8.12.tar.xz&&tar -xf Python-3.8.12.tar.xz")
     .run_commands("cd Python-3.8.12 && ./configure --enable-optimizations && make -j 10 && make altinstall")
-    .add_local_file("heygem-1.0-py3-none-any.whl","/root/heygem-1.0-py3-none-any.whl", copy=True)
+    .add_local_file("../whl/heygem-1.0-py3-none-any.whl", "/root/heygem-1.0-py3-none-any.whl", copy=True)
     .shell(["/bin/bash", "-c"])
     .run_commands("python3.8 -m pip install /root/heygem-1.0-py3-none-any.whl")
     .env({"LD_LIBRARY_PATH":"/usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib"})
     .run_commands("ln -s /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so.11.2 /usr/local/lib/python3.8/site-packages/nvidia/cuda_nvrtc/lib/libnvrtc.so")
     .run_commands("python3.8 -m pip install https://github.com/pydata/numexpr/releases/download/v2.8.6/numexpr-2.8.6-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl")
     .add_local_file("heygem.py", "/root/heygem.py", copy=True)
-    .add_local_file("config.yaml", "/root/config.yaml", copy=True)
-    .workdir("/root").pip_install("boto3").pip_install("PyYAML").pip_install("requests")
+    .pip_install("boto3").pip_install("requests").workdir("/root").add_local_python_source("HeyGem_Base")
 )
 
-app = modal.App(name="heygem", image=image)
+heygem_base_app = modal.App(name="HeyGem-Base", image=heygem_base_image, include_source=False)
+heygem_base_app.set_description("HeyGem Base Server")
+
+
+with heygem_base_image.imports():
+    import json
+    import os
+    import shutil
+    import socket
+    import subprocess
+    import time
+    import traceback
+    import uuid
+    from typing import Optional, Union
+    import httpx
+    import loguru
+    import requests
+    import starlette
+    from fastapi import Depends, HTTPException, status, UploadFile
+    from fastapi.security import HTTPBearer, HTTPAuthorizationCredentials
+
 secret = modal.Secret.from_name("aws-s3-secret")
 auth_scheme = HTTPBearer()
-bucket_output = "bw-heygem-output"
+bucket_output = "modal-media-cache"
 
-@app.cls(
-    allow_concurrent_inputs=1,  # required for UI startup process which runs several API calls concurrently
+@heygem_base_app.cls(
     max_containers=25,  # limit interactive session to 1 container
+    min_containers=0,
+    buffer_containers=0,
+    scaledown_window=240,
+    timeout=2160,
     gpu="L40S",  # good starter GPU for inference
     cpu=(2,32),
     memory=(6144, 32768),
-    timeout=2160,
-    scaledown_window=240,
+    enable_memory_snapshot=False,
     secrets=[secret, modal.Secret.from_name("web_auth_token")],
     volumes={
         "/code/data/final": modal.CloudBucketMount(
             bucket_name=bucket_output,
             # bucket_endpoint_url="https://s3.%s.amazonaws.com" % os.environ["AWS_REGION"],
             secret=secret,
-            key_prefix="/"
+            key_prefix="/heygem/"
         ),
         "/root/logs": modal.CloudBucketMount(
             bucket_name=bucket_output,
             # bucket_endpoint_url="https://s3.%s.amazonaws.com" % os.environ["AWS_REGION"],
             secret=secret,
-            key_prefix="/logs/"
+            key_prefix="/heygem_logs/"
         )
     },  # mounts our cached models
 )
-class HeyGem:
+@modal.concurrent(max_inputs=1)
+class HeyGemBase:
     @modal.enter()
     def start(self):
         def check_port_in_use(port, host='127.0.0.1'):
@@ -232,7 +240,7 @@ class HeyGem:
             with open(path, 'wb') as f:
                 shutil.copyfileobj(r.raw, f)
 
-    @modal.fastapi_endpoint(method="POST")
+    @modal.fastapi_endpoint(method="POST", docs=True)
     async def api(self, video_file: Optional[Union[UploadFile, str]], audio_file: Optional[Union[UploadFile, str]], token: HTTPAuthorizationCredentials = Depends(auth_scheme)):
         if token.credentials != os.environ["AUTH_TOKEN"]:
             raise HTTPException(
@@ -242,7 +250,7 @@ class HeyGem:
             )
         code = str(uuid.uuid4())
         try:
-            if isinstance(video_file, UploadFile):
+            if isinstance(video_file, starlette.datastructures.UploadFile):
                 video_path = "/root/%s" % video_file.filename
                 with open(video_path, "wb") as f:
                     data = await video_file.read()
@@ -253,7 +261,7 @@ class HeyGem:
             else:
                 return {"status": "fail", "msg": "video file save failed: Not a valid input"}
 
-            if isinstance(audio_file, UploadFile):
+            if isinstance(audio_file, starlette.datastructures.UploadFile):
                 audio_path = "/root/%s" % audio_file.filename
                 with open(audio_path, "wb") as f:
                     data = await audio_file.read()
@@ -277,22 +285,12 @@ class HeyGem:
                     print("Try move file to S3 manually")
                     # S3 Fallback
                     import boto3
-                    import yaml
-                    with open("/root/config.yaml", encoding="utf-8",
-                              mode="r+") as config:
-                        yaml_config = yaml.load(config, Loader=yaml.FullLoader)
-                    awss3 = boto3.resource('s3', aws_access_key_id=yaml_config["aws_key_id"],
-                                           aws_secret_access_key=yaml_config["aws_access_key"])
+                    awss3 = boto3.resource('s3', aws_access_key_id=os.environ.get("AWS_ACCESS_KEY_ID"),
+                                           aws_secret_access_key=os.environ.get("AWS_SECRET_ACCESS_KEY"))
                     awss3.meta.client.upload_file(file_path, bucket_output, file_path.split(os.path.sep)[-1])
                 except:
                     return {"status": "fail", "msg": "Failed to move file to S3 manually: " + str(e)}
-            # resp = FileResponse(path=file_path, filename=Path(file_path).name, status_code=200, headers={
-            #     "Content-Type": "application/octet-stream",
-            #     "Content-Disposition": f"attachment; filename={Path(file_path).name}",
-            # })
-            return {"status": "success", "msg":f"{file_path.split(os.path.sep)[-1]}"}
+            return {"status": "success", "url":f"https://cdn.roasmax.cn/heygem/{file_path.split(os.path.sep)[-1]}"}
         except Exception as e:
             traceback.print_exc()
             return {"status": "fail", "msg": "Inference module failed: "+str(e)}
-
-

src/server_cluster/app.py

@@ -0,0 +1,14 @@
+import modal
+
+from ComfyUI_Base.web.worker import comfyui_base_app
+from ComfyUI_Auth.web.worker import comfyui_auth_app
+from ComfyUI_Auth_HeyGem.web.worker import comfyui_auth_heygem_app
+from ComfyUI_Auth_LatentSync1_5.web.worker import comfyui_auth_latentsync_1_5_app
+from HeyGem_Base.web.worker import heygem_base_app
+
+app = modal.App(name="HeyGem-server")
+# app.include(comfyui_base_app)
+# app.include(comfyui_auth_app)
+# app.include(comfyui_auth_heygem_app)
+# app.include(comfyui_auth_latentsync_1_5_app)
+app.include(heygem_base_app)