From 9f25ec0ced742f97b6c41a702183a45f54ba35c3 Mon Sep 17 00:00:00 2001 From: "shuohigh@gmail.com" Date: Thu, 15 May 2025 15:53:37 +0800 Subject: [PATCH] =?UTF-8?q?=E6=96=B0=E5=A2=9EAPI=20-=20/ffmpeg/subtitle-ap?= =?UTF-8?q?ply=20-=20/ffmpeg/bgm-nosie-reduce=20-=20/ffmpeg/zoom-loop=20-?= =?UTF-8?q?=20/ffmpeg/overlay-gif?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 增加modal环境变量.runtime.env --- .runtime.env | 6 ++ pyproject.toml | 1 + src/BowongModalFunctions/api.py | 59 ++++++------- src/BowongModalFunctions/config.py | 24 +++--- src/cluster/app.py | 14 +-- src/cluster/ffmpeg_app.py | 50 +++++------ src/cluster/video.py | 43 +++++----- src/cluster/web.py | 2 + uv.lock | 132 +++++++++++++++-------------- 9 files changed, 167 insertions(+), 164 deletions(-) create mode 100644 .runtime.env diff --git a/.runtime.env b/.runtime.env new file mode 100644 index 0000000..6595d0c --- /dev/null +++ b/.runtime.env @@ -0,0 +1,6 @@ +MODAL_ENVIRONMENT=test +modal_app_name=bowong-ai-video +S3_mount_dir=/mntS3 +S3_bucket_name=modal-media-cache +S3_region=ap-northeast-2 +S3_cdn_endpoint=https://d2nj71io21vkj2.cloudfront.net \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml index 970529b..2412368 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -29,6 +29,7 @@ dependencies = [ "psutil>=7.0.0", "scalar-fastapi>=1.0.3", "modal>=0.76.3", + "python-dotenv>=1.1.0", ] classifiers = [ "Programming Language :: Python :: 3", diff --git a/src/BowongModalFunctions/api.py b/src/BowongModalFunctions/api.py index f44da34..631645e 100644 --- a/src/BowongModalFunctions/api.py +++ b/src/BowongModalFunctions/api.py @@ -47,23 +47,20 @@ from .config import WorkerConfig bearer_scheme = HTTPBearer() -config = WorkerConfig( - app_name="bowong-ai-video", - video_downloader_concurrency=10, - ffmpeg_worker_concurrency=10, - modal_kv_name="media-cache", - s3_region='ap-northeast-2', - s3_bucket_name="modal-media-cache", - environment="dev" -) +config = WorkerConfig() web_app = FastAPI(title="Modal worker API", summary="Modal Worker的API, 包括缓存视频, 发起生产任务等", servers=[ {'url': 'https://bowongai-dev--bowong-ai-video-fastapi-webapp.modal.run', - 'description': 'modal dev环境测试服务'}, + 'description': 'modal 开发环境服务'}, {'url': 'https://modal-dev.bowong.cc', - 'description': 'modal dev环境测试服务'}, + 'description': 'modal 开发环境服务独立域名'}, + {'url': 'https://bowongai-test--bowong-ai-video-fastapi-webapp.modal.run', + 'description': 'modal 测试环境服务'}, + {'url': 'https://bowongai-main--bowong-ai-video-fastapi-webapp.modal.run', + 'description': 'modal 生产环境服务' + } ]) @@ -90,13 +87,13 @@ sentry_sdk.init(dsn="https://dab7b7ae652216282c89f029a76bb10a@sentry.bowongai.co traces_sample_rate=1.0, profiles_sample_rate=1.0, add_full_stack=True, - environment=config.environment, + environment=config.modal_environment, integrations=[ LoguruIntegration(level=LoggingLevels.INFO.value, event_level=LoggingLevels.ERROR.value), FastApiIntegration() ] ) -modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.environment) +modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.modal_environment) cf_account_id = os.environ.get("CF_ACCOUNT_ID") cf_kv_api_token = os.environ.get("CF_KV_API_TOKEN") cf_kv_namespace_id = os.environ.get("CF_KV_NAMESPACE_ID") @@ -244,7 +241,7 @@ async def cache(medias: MediaSources) -> CacheResult: # start new download task with cache_span.start_child(name="视频缓存任务入队", op="queue.publish") as queue_publish_span: - fn = modal.Function.from_name(config.app_name, 'cache_submit') + fn = modal.Function.from_name(config.modal_app_name, 'cache_submit') fn_task = fn.spawn(media, sentry_trace) queue_publish_span.set_data("cache.key", media.urn) queue_publish_span.set_data("messaging.message.id", fn_task.object_id) @@ -269,7 +266,7 @@ async def cache(medias: MediaSources) -> CacheResult: # start new download task with cache_span.start_child(name="视频缓存任务入队", op="queue.publish") as queue_publish_span: - fn = modal.Function.from_name(config.app_name, 'cache_submit') + fn = modal.Function.from_name(config.modal_app_name, 'cache_submit') fn_task = fn.spawn(media, sentry_trace) queue_publish_span.set_data("cache.key", media.urn) queue_publish_span.set_data("messaging.message.id", fn_task.object_id) @@ -305,7 +302,7 @@ async def cache(medias: MediaSources) -> CacheResult: dependencies=[Depends(verify_token)]) @sentry_sdk.trace async def download_caches(medias: MediaSources) -> DownloadResult: - cdn_endpoint = config.cdn_endpoint + cdn_endpoint = config.S3_cdn_endpoint urls = [] for media in medias.inputs: urls.append(f"{cdn_endpoint}/{media.get_cdn_url()}") @@ -318,7 +315,7 @@ async def download_caches(medias: MediaSources) -> DownloadResult: description="通过CDN下载已缓存的视频文件") @sentry_sdk.trace async def download_cache(media: str) -> RedirectResponse: - cdn_endpoint = config.cdn_endpoint + cdn_endpoint = config.S3_cdn_endpoint media = MediaSource.from_str(media) return RedirectResponse(url=f"{cdn_endpoint}/{media.get_cdn_url()}", status_code=status.HTTP_302_FOUND) @@ -360,7 +357,7 @@ async def purge_kv(medias: MediaSources): dependencies=[Depends(verify_token)]) async def purge_media(medias: MediaSources): fn_id = current_function_call_id() - fn = modal.Function.from_name(config.app_name, "cache_delete", environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "cache_delete", environment_name=config.modal_environment) @SentryUtils.sentry_tracker(name="清除媒体源缓存", op="cache.purge", fn_id=fn_id, sentry_trace_id=None, sentry_baggage=None) @@ -386,7 +383,8 @@ async def purge_media(medias: MediaSources): dependencies=[Depends(verify_token)]) async def slice_media(request: FFMPEGSliceRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - fn = modal.Function.from_name(config.app_name, "ffmpeg_slice_media", environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_slice_media", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -422,8 +420,8 @@ async def slice_media(task_id: str, response: Response) -> FFMPEGSliceTaskStatus async def concat_media(body: FFMPEGConcatRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: medias = body.medias - fn = modal.Function.from_name(config.app_name, "ffmpeg_concat_medias", - environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_concat_medias", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -461,8 +459,8 @@ async def concat_media_status(task_id: str, response: Response) -> FFMPEGConcatT async def extract_audio(body: FFMPEGExtractAudioRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: media = body.media - fn = modal.Function.from_name(config.app_name, "ffmpeg_extract_audio", - environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_extract_audio", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -496,8 +494,8 @@ async def extract_audio_status(task_id: str, response: Response) -> FFMPEGExtrac async def corner_mirror(body: FFMPEGCornerMirrorRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: media = body.media - fn = modal.Function.from_name(config.app_name, "ffmpeg_corner_mirror", - environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_corner_mirror", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -529,7 +527,8 @@ async def corner_mirror_status(task_id: str, response: Response) -> FFMPEGCorner description="发起叠加gif特效任务", dependencies=[Depends(verify_token)]) async def overlay_gif(body: FFMPEGOverlayGifRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - fn = modal.Function.from_name(config.app_name, "ffmpeg_overlay_gif", environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_overlay_gif", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -559,7 +558,7 @@ async def overlay_gif_status(task_id: str, response: Response) -> FFMPEGOverlayG description="发起放大缩小循环去重任务", dependencies=[Depends(verify_token)]) async def zoom_loop(body: FFMPEGZoomLoopRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - fn = modal.Function.from_name(config.app_name, "ffmpeg_zoom_loop", environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_zoom_loop", environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -589,7 +588,8 @@ async def zoom_loop_status(task_id: str, response: Response) -> FFMPEGZoomLoopTa description="发起渲染字幕任务", dependencies=[Depends(verify_token)]) async def subtitle_apply(body: FFMPEGSubtitleOverlayRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - fn = modal.Function.from_name(config.app_name, "ffmpeg_subtitle_apply", environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_subtitle_apply", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) @@ -619,7 +619,8 @@ async def subtitle_apply_status(task_id: str, response: Response) -> FFMPEGSubti description="发起混合BGM并降噪任务", dependencies=[Depends(verify_token)]) async def bgm_nosie_reduce(body: FFMPEGMixBgmWithNoiseReduceRequest, headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse: - fn = modal.Function.from_name(config.app_name, "ffmpeg_bgm_nosie_reduce", environment_name=config.environment) + fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_bgm_nosie_reduce", + environment_name=config.modal_environment) sentry_trace = None if headers.x_trace_id and headers.x_baggage: sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage) diff --git a/src/BowongModalFunctions/config.py b/src/BowongModalFunctions/config.py index 85ad579..9bff9c7 100644 --- a/src/BowongModalFunctions/config.py +++ b/src/BowongModalFunctions/config.py @@ -2,17 +2,19 @@ from typing import Optional, Any from pydantic import Field from pydantic_settings import BaseSettings, SettingsConfigDict -class WorkerConfig(BaseSettings): - app_name: str = Field(default='bowong-ai-video', description="Modal App集群名称") - video_downloader_concurrency: Optional[int] = Field(default=10, description="处理缓存任务的并行数") - ffmpeg_worker_concurrency: Optional[int] = Field(default=10, description="处理视频合成任务的并行数") - modal_kv_name: Optional[str] = Field(default='media-cache', description="Modal视频缓存KV库") - s3_region: Optional[str] = Field(default='ap-northeast-2', description="S3挂载桶的地域") - s3_bucket_name: Optional[str] = Field(default='modal-media-cache', description="集群挂载的S3存储桶") - s3_mount_dir: str = Field(default='/mntS3', description="集群S3存储桶挂载在本地的根目录") - cdn_endpoint: Optional[str] = Field(default="https://d2nj71io21vkj2.cloudfront.net", - description="集群挂载S3存储桶的对应AWS Cloudfront CDN") - environment: Optional[str] = Field(default="dev", description="Modal worker运行环境") +class WorkerConfig(BaseSettings): + video_downloader_concurrency: int = Field(default=10, description="处理缓存任务的并行数") + ffmpeg_worker_concurrency: int = Field(default=10, description="处理视频合成任务的并行数") + + S3_region: str = Field(default='ap-northeast-2', description="S3挂载桶的地域") + S3_bucket_name: str = Field(default='modal-media-cache', description="集群挂载的S3存储桶") + S3_mount_dir: str = Field(default='/mntS3', description="集群S3存储桶挂载在本地的根目录") + S3_cdn_endpoint: str = Field(default="https://d2nj71io21vkj2.cloudfront.net", + description="集群挂载S3存储桶的对应AWS Cloudfront CDN") + + modal_kv_name: str = Field(default='media-cache', description="Modal视频缓存KV库") + modal_environment: str = Field(default="dev", description="Modal worker运行环境") + modal_app_name: str = Field(default='bowong-ai-video', description="Modal App集群名称") modal_config: Any = SettingsConfigDict() diff --git a/src/cluster/app.py b/src/cluster/app.py index d0d79f2..20f5ca7 100644 --- a/src/cluster/app.py +++ b/src/cluster/app.py @@ -4,20 +4,12 @@ from .video import app as media_app from .web import app as web_app from .ffmpeg_app import app as ffmpeg_app -config = WorkerConfig( - app_name="bowong-ai-video-test", - video_downloader_concurrency=10, - ffmpeg_worker_concurrency=10, - modal_kv_name="media-cache", - s3_region='ap-northeast-2', - s3_bucket_name="modal-media-cache", - environment="dev" -) +config = WorkerConfig() -app = modal.App('bowong-modal-ai-video-preview', +app = modal.App(config.modal_app_name, include_source=False, secrets=[modal.Secret.from_name("cf-kv-secret", - environment_name=config.environment)]) + environment_name=config.modal_environment)]) app.include(media_app) app.include(ffmpeg_app) diff --git a/src/cluster/ffmpeg_app.py b/src/cluster/ffmpeg_app.py index 81c3500..d4a473e 100644 --- a/src/cluster/ffmpeg_app.py +++ b/src/cluster/ffmpeg_app.py @@ -1,9 +1,11 @@ import modal +from dotenv import dotenv_values ffmpeg_worker_image = ( modal.Image.debian_slim(python_version="3.11") .apt_install('ffmpeg') .pip_install_from_pyproject("pyproject.toml") + .env(dotenv_values("../../.runtime.env")) .add_local_python_source('cluster') .add_local_python_source('BowongModalFunctions') ) @@ -27,17 +29,9 @@ with ffmpeg_worker_image.imports(): from BowongModalFunctions.models.web_model import SentryTransactionInfo from BowongModalFunctions.config import WorkerConfig - config = WorkerConfig( - app_name="bowong-ai-video-test", - video_downloader_concurrency=10, - ffmpeg_worker_concurrency=10, - modal_kv_name="media-cache", - s3_region='ap-northeast-2', - s3_bucket_name="modal-media-cache", - environment="dev" - ) + config = WorkerConfig() - s3_mount = config.s3_mount_dir + s3_mount = config.S3_mount_dir output_path_prefix = "/mnt/outputs" sentry_sdk.init( @@ -46,7 +40,7 @@ with ffmpeg_worker_image.imports(): add_full_stack=True, shutdown_timeout=2, traces_sample_rate=1.0, - environment=config.environment, + environment=config.modal_environment, ) @@ -107,7 +101,7 @@ with ffmpeg_worker_image.imports(): os.makedirs(out_s3_dir, exist_ok=True) shutil.copy(output, out_s3) s3_outputs.append( - out_s3.replace(s3_mount + '/', f"s3://{config.s3_region}/{config.s3_bucket_name}/")) + out_s3.replace(s3_mount + '/', f"s3://{config.S3_region}/{config.S3_bucket_name}/")) return s3_outputs @@ -118,9 +112,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @modal.concurrent(max_inputs=1) @@ -154,9 +148,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @@ -275,9 +269,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @@ -311,9 +305,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @modal.concurrent(max_inputs=1) @@ -353,9 +347,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @modal.concurrent(max_inputs=1) @@ -387,9 +381,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @modal.concurrent(max_inputs=1) @@ -421,9 +415,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @modal.concurrent(max_inputs=1) @@ -464,9 +458,9 @@ with ffmpeg_worker_image.imports(): max_containers=config.ffmpeg_worker_concurrency, volumes={ f"/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, + bucket_name=config.S3_bucket_name, secret=modal.Secret.from_name("aws-s3-secret", - environment_name=config.environment), + environment_name=config.modal_environment), ), }, ) @modal.concurrent(max_inputs=1) diff --git a/src/cluster/video.py b/src/cluster/video.py index 5ceaf36..65f272a 100644 --- a/src/cluster/video.py +++ b/src/cluster/video.py @@ -1,9 +1,11 @@ import modal +from dotenv import dotenv_values downloader_image = ( modal.Image .debian_slim(python_version="3.11") .pip_install_from_pyproject("pyproject.toml") + .env(dotenv_values("../../.runtime.env")) .add_local_python_source('cluster') .add_local_python_source('BowongModalFunctions') ) @@ -35,15 +37,16 @@ with downloader_image.imports(): from BowongModalFunctions.models.media_model import MediaSource, MediaCacheStatus, MediaProtocol from BowongModalFunctions.models.web_model import SentryTransactionInfo - config = WorkerConfig( - app_name="bowong-ai-video-test", - video_downloader_concurrency=10, - ffmpeg_worker_concurrency=10, - modal_kv_name="media-cache", - s3_region='ap-northeast-2', - s3_bucket_name="modal-media-cache", - environment="dev" - ) + config = WorkerConfig() + # config = WorkerConfig( + # modal_app_name="bowong-ai-video-test", + # video_downloader_concurrency=10, + # ffmpeg_worker_concurrency=10, + # modal_kv_name="media-cache", + # S3_region='ap-northeast-2', + # S3_bucket_name="modal-media-cache", + # modal_environment="dev" + # ) sentry_sdk.init(dsn="https://85632fdcd62f699c2f88af6ca489e9ec@sentry.bowongai.com/3", send_default_pii=True, @@ -52,14 +55,14 @@ with downloader_image.imports(): add_full_stack=True, shutdown_timeout=2, integrations=[LoguruIntegration()], - environment=config.environment, + environment=config.modal_environment, ) cf_account_id = os.environ.get("CF_ACCOUNT_ID") cf_kv_api_token = os.environ.get("CF_KV_API_TOKEN") cf_kv_namespace_id = os.environ.get("CF_KV_NAMESPACE_ID") - modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.environment) + modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.modal_environment) @sentry_sdk.trace @@ -117,11 +120,11 @@ with downloader_image.imports(): max_containers=config.video_downloader_concurrency, volumes={ "/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, - secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment), + bucket_name=config.S3_bucket_name, + secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment), ), }, - secrets=[modal.Secret.from_name("tencent-cloud-secret", environment_name=config.environment)]) + secrets=[modal.Secret.from_name("tencent-cloud-secret", environment_name=config.modal_environment)]) @modal.concurrent(max_inputs=10) async def cache_submit(media: MediaSource, sentry_trace: SentryTransactionInfo) -> MediaSource: def vod_init(): @@ -141,7 +144,7 @@ with downloader_image.imports(): media_info = response.MediaInfoSet[0].BasicInfo logger.info(f"VOD info = {media_info}") file_extension = media_info.Type - cache_dir = f"/{config.s3_mount_dir}/{media.protocol.value}/{media.endpoint}/{media.bucket}" + cache_dir = f"/{config.S3_mount_dir}/{media.protocol.value}/{media.endpoint}/{media.bucket}" cache_file = media.path if '.' in media.path else f"{media.path}.{file_extension}" return (cache_dir, cache_file, media_info.MediaUrl) else: @@ -280,7 +283,7 @@ with downloader_image.imports(): process_span.set_status("failed") case MediaProtocol.http: try: - cache_filepath = f"{config.s3_mount_dir}/{media.cache_filepath}" + cache_filepath = f"{config.S3_mount_dir}/{media.cache_filepath}" download_large_file(url=media.__str__(), output_path=cache_filepath) except Exception as e: logger.exception(e) @@ -290,8 +293,8 @@ with downloader_image.imports(): process_span.set_status("failed") case MediaProtocol.s3: # 本地挂载缓存 - if media.protocol == MediaProtocol.s3 and media.endpoint == config.s3_region and media.bucket == config.s3_bucket_name: - volume_cache_path = f"{config.s3_mount_dir}/{media.cache_filepath}" + if media.protocol == MediaProtocol.s3 and media.endpoint == config.S3_region and media.bucket == config.S3_bucket_name: + volume_cache_path = f"{config.S3_mount_dir}/{media.cache_filepath}" else: logger.error("protocol not yet supported") case _: @@ -310,8 +313,8 @@ with downloader_image.imports(): max_containers=config.video_downloader_concurrency, volumes={ "/mntS3": modal.CloudBucketMount( - bucket_name=config.s3_bucket_name, - secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment), + bucket_name=config.S3_bucket_name, + secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment), ), }) @modal.concurrent(max_inputs=10) diff --git a/src/cluster/web.py b/src/cluster/web.py index c4c9efc..8166fd3 100644 --- a/src/cluster/web.py +++ b/src/cluster/web.py @@ -1,9 +1,11 @@ import modal +from dotenv import dotenv_values fastapi_image = ( modal.Image .debian_slim(python_version="3.11") .pip_install_from_pyproject("pyproject.toml") + .env(dotenv_values("../../.runtime.env")) .add_local_python_source('cluster') .add_local_python_source('BowongModalFunctions') ) diff --git a/uv.lock b/uv.lock index 899ecc6..6130918 100644 --- a/uv.lock +++ b/uv.lock @@ -166,6 +166,67 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f9/92/2c522e277c95d35b4b83bff6a3839875d91b0d835a93545828a7046013c4/botocore-1.38.10-py3-none-any.whl", hash = "sha256:5244454bb9e8fbb6510145d1554e82fd243e8583507d83077ecf4f8efb66cb46", size = 13539530 }, ] +[[package]] +name = "bowongmodalfunctions" +source = { editable = "." } +dependencies = [ + { name = "backoff" }, + { name = "boto3" }, + { name = "cos-python-sdk-v5" }, + { name = "crcmod" }, + { name = "fastapi", extra = ["standard"] }, + { name = "httpx" }, + { name = "loguru" }, + { name = "modal" }, + { name = "noisereduce" }, + { name = "pedalboard" }, + { name = "psutil" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "pyloudnorm" }, + { name = "python-dotenv" }, + { name = "python-ffmpeg" }, + { name = "scalar-fastapi" }, + { name = "sentry-sdk", extra = ["loguru"] }, + { name = "soundfile" }, + { name = "tencentcloud-sdk-python-common" }, + { name = "tencentcloud-sdk-python-vod" }, + { name = "webvtt-py" }, +] + +[package.optional-dependencies] +cli = [ + { name = "twine" }, +] + +[package.metadata] +requires-dist = [ + { name = "backoff", specifier = ">=2.2.1" }, + { name = "boto3", specifier = ">=1.37.37" }, + { name = "cos-python-sdk-v5", specifier = ">=1.9.36" }, + { name = "crcmod", specifier = ">=1.7" }, + { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" }, + { name = "httpx", specifier = ">=0.28.1" }, + { name = "loguru", specifier = ">=0.7.3" }, + { name = "modal", specifier = ">=0.76.3" }, + { name = "noisereduce", specifier = ">=3.0.3" }, + { name = "pedalboard", specifier = "==0.9.2" }, + { name = "psutil", specifier = ">=7.0.0" }, + { name = "pydantic", specifier = ">=2.11.3" }, + { name = "pydantic-settings", specifier = ">=2.9.1" }, + { name = "pyloudnorm", specifier = ">=0.1.1" }, + { name = "python-dotenv", specifier = ">=1.1.0" }, + { name = "python-ffmpeg", specifier = ">=2.0.12" }, + { name = "scalar-fastapi", specifier = ">=1.0.3" }, + { name = "sentry-sdk", extras = ["loguru"], specifier = ">=2.26.1" }, + { name = "soundfile", specifier = ">=0.13.1" }, + { name = "tencentcloud-sdk-python-common", specifier = ">=3.0.1363" }, + { name = "tencentcloud-sdk-python-vod", specifier = ">=3.0.1363" }, + { name = "twine", marker = "extra == 'cli'", specifier = ">=6.1.0" }, + { name = "webvtt-py", specifier = ">=0.5.1" }, +] +provides-extras = ["cli"] + [[package]] name = "certifi" version = "2025.4.26" @@ -1023,7 +1084,7 @@ wheels = [ [[package]] name = "modal" -version = "0.74.57" +version = "0.77.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "aiohttp" }, @@ -1040,68 +1101,9 @@ dependencies = [ { name = "typing-extensions" }, { name = "watchfiles" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/93/6e/ac879223fbe05b0dae9eb8bf854d5b2e635638929b71a6332cc291972a6f/modal-0.74.57.tar.gz", hash = "sha256:5d598eee638da862a574a433ba55b83ed1ab617e802982c6ff589913307b7a77", size = 505567 } +sdist = { url = "https://files.pythonhosted.org/packages/32/06/acd53f8fb8d4f0f7a1d5a5bfdb962fcbbe917ed6f703d2a6dfdbfb490c34/modal-0.77.0.tar.gz", hash = "sha256:0a3c2b438a5d26c4b2a1100a7cecdd47e09f715e2a6e0de2a50aad235ddc92c3", size = 508716 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2d/8d/92b981506e3ce3102eb36f2cfab2e980ed0930bf6b39023d9a4896cb460a/modal-0.74.57-py3-none-any.whl", hash = "sha256:a576c2aa9db6c47e8ce3e36f5159d77e4fa807676d9ec22f0a92c9d92ad2bf74", size = 573992 }, -] - -[[package]] -name = "modaldeploy" -version = "0.2.0" -source = { virtual = "." } -dependencies = [ - { name = "backoff" }, - { name = "boto3" }, - { name = "cos-python-sdk-v5" }, - { name = "crcmod" }, - { name = "fastapi", extra = ["standard"] }, - { name = "httpx" }, - { name = "loguru" }, - { name = "modal" }, - { name = "noisereduce" }, - { name = "pedalboard" }, - { name = "psutil" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyloudnorm" }, - { name = "python-ffmpeg" }, - { name = "requests" }, - { name = "scalar-fastapi" }, - { name = "sentry-sdk", extra = ["loguru"] }, - { name = "soundfile" }, - { name = "tencentcloud-sdk-python-common" }, - { name = "tencentcloud-sdk-python-vod" }, - { name = "tqdm" }, - { name = "twine" }, - { name = "webvtt-py" }, -] - -[package.metadata] -requires-dist = [ - { name = "backoff", specifier = ">=2.2.1" }, - { name = "boto3", specifier = ">=1.37.37" }, - { name = "cos-python-sdk-v5", specifier = ">=1.9.36" }, - { name = "crcmod", specifier = ">=1.7" }, - { name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" }, - { name = "httpx", specifier = ">=0.28.1" }, - { name = "loguru", specifier = ">=0.7.3" }, - { name = "modal", specifier = ">=0.74.57" }, - { name = "noisereduce", specifier = ">=3.0.3" }, - { name = "pedalboard", specifier = "==0.9.2" }, - { name = "psutil", specifier = ">=7.0.0" }, - { name = "pydantic", specifier = ">=2.11.3" }, - { name = "pydantic-settings", specifier = ">=2.9.1" }, - { name = "pyloudnorm", specifier = ">=0.1.1" }, - { name = "python-ffmpeg", specifier = ">=2.0.12" }, - { name = "requests", specifier = ">=2.32.3" }, - { name = "scalar-fastapi", specifier = ">=1.0.3" }, - { name = "sentry-sdk", extras = ["loguru"], specifier = ">=2.26.1" }, - { name = "soundfile", specifier = ">=0.13.1" }, - { name = "tencentcloud-sdk-python-common", specifier = ">=3.0.1363" }, - { name = "tencentcloud-sdk-python-vod", specifier = ">=3.0.1363" }, - { name = "tqdm", specifier = ">=4.67.1" }, - { name = "twine", specifier = ">=6.1.0" }, - { name = "webvtt-py", specifier = ">=0.5.1" }, + { url = "https://files.pythonhosted.org/packages/db/a2/06b274bbec3501de530e8be5da02e62258f568201200bcb115976ae3b68e/modal-0.77.0-py3-none-any.whl", hash = "sha256:f646d47e9950b929a03918124c4b12663fdc146287af3c0bd96de3ce23776399", size = 576782 }, ] [[package]] @@ -1981,15 +1983,15 @@ wheels = [ [[package]] name = "synchronicity" -version = "0.9.11" +version = "0.9.12" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "sigtools" }, { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/b5/52/f34a9ab6d514e0808d0f572affb360411d596b3439107318c00889277dd6/synchronicity-0.9.11.tar.gz", hash = "sha256:cb5dbbcb43d637e516ae50db05a776da51a705d1e1a9c0e301f6049afc3c2cae", size = 50323 } +sdist = { url = "https://files.pythonhosted.org/packages/ed/21/82c2b51f901452642cb2ed69ab872ef5eacbf98dbd0b661d3673a76d79f5/synchronicity-0.9.12.tar.gz", hash = "sha256:977f3ed8f6e35de4d1a3f0aeee4937143ba8d913f531d33e8df7c539b2792fb8", size = 50441 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f2/d5/7675cd9b8e18f05b9ea261acad5d197fcb8027d2a65b1a750427ec084593/synchronicity-0.9.11-py3-none-any.whl", hash = "sha256:231129654d2f56b1aa148e85ebd8545231be135771f6d2196d414175b1594ef6", size = 36827 }, + { url = "https://files.pythonhosted.org/packages/ab/e3/730a67bee380f5638731cd563e8f5ad3a2d480ee788b767698e83eb2290f/synchronicity-0.9.12-py3-none-any.whl", hash = "sha256:b006f57bd216d55e578316096a11b6dc16016d6b48e2766bcffabe40c88f9793", size = 36820 }, ] [[package]]