bowong
/
modalDeploy
6
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

新增API 

- /ffmpeg/subtitle-apply
- /ffmpeg/bgm-nosie-reduce
- /ffmpeg/zoom-loop
- /ffmpeg/overlay-gif

增加modal环境变量.runtime.env
This commit is contained in:
shuohigh@gmail.com 2025-05-15 15:53:37 +08:00
parent 0fca52a271
commit 9f25ec0ced
9 changed files with 167 additions and 164 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.runtime.env Normal file
View File

@ -0,0 +1,6 @@
MODAL_ENVIRONMENT=test
modal_app_name=bowong-ai-video
S3_mount_dir=/mntS3
S3_bucket_name=modal-media-cache
S3_region=ap-northeast-2
S3_cdn_endpoint=https://d2nj71io21vkj2.cloudfront.net

1
pyproject.toml
View File

@ -29,6 +29,7 @@ dependencies = [
"psutil>=7.0.0",
"scalar-fastapi>=1.0.3",
"modal>=0.76.3",
"python-dotenv>=1.1.0",
]
classifiers = [
"Programming Language :: Python :: 3",

59
src/BowongModalFunctions/api.py
View File

@ -47,23 +47,20 @@ from .config import WorkerConfig
bearer_scheme = HTTPBearer()
config = WorkerConfig(
app_name="bowong-ai-video",
video_downloader_concurrency=10,
ffmpeg_worker_concurrency=10,
modal_kv_name="media-cache",
s3_region='ap-northeast-2',
s3_bucket_name="modal-media-cache",
environment="dev"
)
config = WorkerConfig()
web_app = FastAPI(title="Modal worker API",
summary="Modal Worker的API, 包括缓存视频, 发起生产任务等",
servers=[
{'url': 'https://bowongai-dev--bowong-ai-video-fastapi-webapp.modal.run',
'description': 'modal dev环境测试服务'},
'description': 'modal 开发环境服务'},
{'url': 'https://modal-dev.bowong.cc',
'description': 'modal dev环境测试服务'},
'description': 'modal 开发环境服务独立域名'},
{'url': 'https://bowongai-test--bowong-ai-video-fastapi-webapp.modal.run',
'description': 'modal 测试环境服务'},
{'url': 'https://bowongai-main--bowong-ai-video-fastapi-webapp.modal.run',
'description': 'modal 生产环境服务'
}
])
@ -90,13 +87,13 @@ sentry_sdk.init(dsn="https://dab7b7ae652216282c89f029a76bb10a@sentry.bowongai.co
traces_sample_rate=1.0,
profiles_sample_rate=1.0,
add_full_stack=True,
environment=config.environment,
environment=config.modal_environment,
integrations=[
LoguruIntegration(level=LoggingLevels.INFO.value, event_level=LoggingLevels.ERROR.value),
FastApiIntegration()
]
)
modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.environment)
modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.modal_environment)
cf_account_id = os.environ.get("CF_ACCOUNT_ID")
cf_kv_api_token = os.environ.get("CF_KV_API_TOKEN")
cf_kv_namespace_id = os.environ.get("CF_KV_NAMESPACE_ID")
@ -244,7 +241,7 @@ async def cache(medias: MediaSources) -> CacheResult:
# start new download task
with cache_span.start_child(name="视频缓存任务入队",
op="queue.publish") as queue_publish_span:
fn = modal.Function.from_name(config.app_name, 'cache_submit')
fn = modal.Function.from_name(config.modal_app_name, 'cache_submit')
fn_task = fn.spawn(media, sentry_trace)
queue_publish_span.set_data("cache.key", media.urn)
queue_publish_span.set_data("messaging.message.id", fn_task.object_id)
@ -269,7 +266,7 @@ async def cache(medias: MediaSources) -> CacheResult:
# start new download task
with cache_span.start_child(name="视频缓存任务入队",
op="queue.publish") as queue_publish_span:
fn = modal.Function.from_name(config.app_name, 'cache_submit')
fn = modal.Function.from_name(config.modal_app_name, 'cache_submit')
fn_task = fn.spawn(media, sentry_trace)
queue_publish_span.set_data("cache.key", media.urn)
queue_publish_span.set_data("messaging.message.id", fn_task.object_id)
@ -305,7 +302,7 @@ async def cache(medias: MediaSources) -> CacheResult:
dependencies=[Depends(verify_token)])
@sentry_sdk.trace
async def download_caches(medias: MediaSources) -> DownloadResult:
cdn_endpoint = config.cdn_endpoint
cdn_endpoint = config.S3_cdn_endpoint
urls = []
for media in medias.inputs:
urls.append(f"{cdn_endpoint}/{media.get_cdn_url()}")
@ -318,7 +315,7 @@ async def download_caches(medias: MediaSources) -> DownloadResult:
description="通过CDN下载已缓存的视频文件")
@sentry_sdk.trace
async def download_cache(media: str) -> RedirectResponse:
cdn_endpoint = config.cdn_endpoint
cdn_endpoint = config.S3_cdn_endpoint
media = MediaSource.from_str(media)
return RedirectResponse(url=f"{cdn_endpoint}/{media.get_cdn_url()}", status_code=status.HTTP_302_FOUND)
@ -360,7 +357,7 @@ async def purge_kv(medias: MediaSources):
dependencies=[Depends(verify_token)])
async def purge_media(medias: MediaSources):
fn_id = current_function_call_id()
fn = modal.Function.from_name(config.app_name, "cache_delete", environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "cache_delete", environment_name=config.modal_environment)
@SentryUtils.sentry_tracker(name="清除媒体源缓存", op="cache.purge", fn_id=fn_id,
sentry_trace_id=None, sentry_baggage=None)
@ -386,7 +383,8 @@ async def purge_media(medias: MediaSources):
dependencies=[Depends(verify_token)])
async def slice_media(request: FFMPEGSliceRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
fn = modal.Function.from_name(config.app_name, "ffmpeg_slice_media", environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_slice_media",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -422,8 +420,8 @@ async def slice_media(task_id: str, response: Response) -> FFMPEGSliceTaskStatus
async def concat_media(body: FFMPEGConcatRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
medias = body.medias
fn = modal.Function.from_name(config.app_name, "ffmpeg_concat_medias",
environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_concat_medias",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -461,8 +459,8 @@ async def concat_media_status(task_id: str, response: Response) -> FFMPEGConcatT
async def extract_audio(body: FFMPEGExtractAudioRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
media = body.media
fn = modal.Function.from_name(config.app_name, "ffmpeg_extract_audio",
environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_extract_audio",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -496,8 +494,8 @@ async def extract_audio_status(task_id: str, response: Response) -> FFMPEGExtrac
async def corner_mirror(body: FFMPEGCornerMirrorRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
media = body.media
fn = modal.Function.from_name(config.app_name, "ffmpeg_corner_mirror",
environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_corner_mirror",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -529,7 +527,8 @@ async def corner_mirror_status(task_id: str, response: Response) -> FFMPEGCorner
description="发起叠加gif特效任务", dependencies=[Depends(verify_token)])
async def overlay_gif(body: FFMPEGOverlayGifRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
fn = modal.Function.from_name(config.app_name, "ffmpeg_overlay_gif", environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_overlay_gif",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -559,7 +558,7 @@ async def overlay_gif_status(task_id: str, response: Response) -> FFMPEGOverlayG
description="发起放大缩小循环去重任务", dependencies=[Depends(verify_token)])
async def zoom_loop(body: FFMPEGZoomLoopRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
fn = modal.Function.from_name(config.app_name, "ffmpeg_zoom_loop", environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_zoom_loop", environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -589,7 +588,8 @@ async def zoom_loop_status(task_id: str, response: Response) -> FFMPEGZoomLoopTa
description="发起渲染字幕任务", dependencies=[Depends(verify_token)])
async def subtitle_apply(body: FFMPEGSubtitleOverlayRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
fn = modal.Function.from_name(config.app_name, "ffmpeg_subtitle_apply", environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_subtitle_apply",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)
@ -619,7 +619,8 @@ async def subtitle_apply_status(task_id: str, response: Response) -> FFMPEGSubti
description="发起混合BGM并降噪任务", dependencies=[Depends(verify_token)])
async def bgm_nosie_reduce(body: FFMPEGMixBgmWithNoiseReduceRequest,
headers: Annotated[SentryTransactionHeader, Header()]) -> ModalTaskResponse:
fn = modal.Function.from_name(config.app_name, "ffmpeg_bgm_nosie_reduce", environment_name=config.environment)
fn = modal.Function.from_name(config.modal_app_name, "ffmpeg_bgm_nosie_reduce",
environment_name=config.modal_environment)
sentry_trace = None
if headers.x_trace_id and headers.x_baggage:
sentry_trace = SentryTransactionInfo(x_trace_id=headers.x_trace_id, x_baggage=headers.x_baggage)

24
src/BowongModalFunctions/config.py
View File

@ -2,17 +2,19 @@ from typing import Optional, Any
from pydantic import Field
from pydantic_settings import BaseSettings, SettingsConfigDict
class WorkerConfig(BaseSettings):
app_name: str = Field(default='bowong-ai-video', description="Modal App集群名称")
video_downloader_concurrency: Optional[int] = Field(default=10, description="处理缓存任务的并行数")
ffmpeg_worker_concurrency: Optional[int] = Field(default=10, description="处理视频合成任务的并行数")
modal_kv_name: Optional[str] = Field(default='media-cache', description="Modal视频缓存KV库")
s3_region: Optional[str] = Field(default='ap-northeast-2', description="S3挂载桶的地域")
s3_bucket_name: Optional[str] = Field(default='modal-media-cache', description="集群挂载的S3存储桶")
s3_mount_dir: str = Field(default='/mntS3', description="集群S3存储桶挂载在本地的根目录")
cdn_endpoint: Optional[str] = Field(default="https://d2nj71io21vkj2.cloudfront.net",
description="集群挂载S3存储桶的对应AWS Cloudfront CDN")
environment: Optional[str] = Field(default="dev", description="Modal worker运行环境")
class WorkerConfig(BaseSettings):
video_downloader_concurrency: int = Field(default=10, description="处理缓存任务的并行数")
ffmpeg_worker_concurrency: int = Field(default=10, description="处理视频合成任务的并行数")
S3_region: str = Field(default='ap-northeast-2', description="S3挂载桶的地域")
S3_bucket_name: str = Field(default='modal-media-cache', description="集群挂载的S3存储桶")
S3_mount_dir: str = Field(default='/mntS3', description="集群S3存储桶挂载在本地的根目录")
S3_cdn_endpoint: str = Field(default="https://d2nj71io21vkj2.cloudfront.net",
description="集群挂载S3存储桶的对应AWS Cloudfront CDN")
modal_kv_name: str = Field(default='media-cache', description="Modal视频缓存KV库")
modal_environment: str = Field(default="dev", description="Modal worker运行环境")
modal_app_name: str = Field(default='bowong-ai-video', description="Modal App集群名称")
modal_config: Any = SettingsConfigDict()

14
src/cluster/app.py
View File

@ -4,20 +4,12 @@ from .video import app as media_app
from .web import app as web_app
from .ffmpeg_app import app as ffmpeg_app
config = WorkerConfig(
app_name="bowong-ai-video-test",
video_downloader_concurrency=10,
ffmpeg_worker_concurrency=10,
modal_kv_name="media-cache",
s3_region='ap-northeast-2',
s3_bucket_name="modal-media-cache",
environment="dev"
)
config = WorkerConfig()
app = modal.App('bowong-modal-ai-video-preview',
app = modal.App(config.modal_app_name,
include_source=False,
secrets=[modal.Secret.from_name("cf-kv-secret",
environment_name=config.environment)])
environment_name=config.modal_environment)])
app.include(media_app)
app.include(ffmpeg_app)

50
src/cluster/ffmpeg_app.py
View File

@ -1,9 +1,11 @@
import modal
from dotenv import dotenv_values
ffmpeg_worker_image = (
modal.Image.debian_slim(python_version="3.11")
.apt_install('ffmpeg')
.pip_install_from_pyproject("pyproject.toml")
.env(dotenv_values("../../.runtime.env"))
.add_local_python_source('cluster')
.add_local_python_source('BowongModalFunctions')
)
@ -27,17 +29,9 @@ with ffmpeg_worker_image.imports():
from BowongModalFunctions.models.web_model import SentryTransactionInfo
from BowongModalFunctions.config import WorkerConfig
config = WorkerConfig(
app_name="bowong-ai-video-test",
video_downloader_concurrency=10,
ffmpeg_worker_concurrency=10,
modal_kv_name="media-cache",
s3_region='ap-northeast-2',
s3_bucket_name="modal-media-cache",
environment="dev"
)
config = WorkerConfig()
s3_mount = config.s3_mount_dir
s3_mount = config.S3_mount_dir
output_path_prefix = "/mnt/outputs"
sentry_sdk.init(
@ -46,7 +40,7 @@ with ffmpeg_worker_image.imports():
add_full_stack=True,
shutdown_timeout=2,
traces_sample_rate=1.0,
environment=config.environment,
environment=config.modal_environment,
)
@ -107,7 +101,7 @@ with ffmpeg_worker_image.imports():
os.makedirs(out_s3_dir, exist_ok=True)
shutil.copy(output, out_s3)
s3_outputs.append(
out_s3.replace(s3_mount + '/', f"s3://{config.s3_region}/{config.s3_bucket_name}/"))
out_s3.replace(s3_mount + '/', f"s3://{config.S3_region}/{config.S3_bucket_name}/"))
return s3_outputs
@ -118,9 +112,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
}, )
@modal.concurrent(max_inputs=1)
@ -154,9 +148,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
},
)
@ -275,9 +269,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
},
)
@ -311,9 +305,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
}, )
@modal.concurrent(max_inputs=1)
@ -353,9 +347,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
}, )
@modal.concurrent(max_inputs=1)
@ -387,9 +381,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
}, )
@modal.concurrent(max_inputs=1)
@ -421,9 +415,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
}, )
@modal.concurrent(max_inputs=1)
@ -464,9 +458,9 @@ with ffmpeg_worker_image.imports():
max_containers=config.ffmpeg_worker_concurrency,
volumes={
f"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret",
environment_name=config.environment),
environment_name=config.modal_environment),
),
}, )
@modal.concurrent(max_inputs=1)

43
src/cluster/video.py
View File

@ -1,9 +1,11 @@
import modal
from dotenv import dotenv_values
downloader_image = (
modal.Image
.debian_slim(python_version="3.11")
.pip_install_from_pyproject("pyproject.toml")
.env(dotenv_values("../../.runtime.env"))
.add_local_python_source('cluster')
.add_local_python_source('BowongModalFunctions')
)
@ -35,15 +37,16 @@ with downloader_image.imports():
from BowongModalFunctions.models.media_model import MediaSource, MediaCacheStatus, MediaProtocol
from BowongModalFunctions.models.web_model import SentryTransactionInfo
config = WorkerConfig(
app_name="bowong-ai-video-test",
video_downloader_concurrency=10,
ffmpeg_worker_concurrency=10,
modal_kv_name="media-cache",
s3_region='ap-northeast-2',
s3_bucket_name="modal-media-cache",
environment="dev"
)
config = WorkerConfig()
# config = WorkerConfig(
# modal_app_name="bowong-ai-video-test",
# video_downloader_concurrency=10,
# ffmpeg_worker_concurrency=10,
# modal_kv_name="media-cache",
# S3_region='ap-northeast-2',
# S3_bucket_name="modal-media-cache",
# modal_environment="dev"
# )
sentry_sdk.init(dsn="https://85632fdcd62f699c2f88af6ca489e9ec@sentry.bowongai.com/3",
send_default_pii=True,
@ -52,14 +55,14 @@ with downloader_image.imports():
add_full_stack=True,
shutdown_timeout=2,
integrations=[LoguruIntegration()],
environment=config.environment,
environment=config.modal_environment,
)
cf_account_id = os.environ.get("CF_ACCOUNT_ID")
cf_kv_api_token = os.environ.get("CF_KV_API_TOKEN")
cf_kv_namespace_id = os.environ.get("CF_KV_NAMESPACE_ID")
modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.environment)
modal_kv_cache = KVCache(kv_name=config.modal_kv_name, environment=config.modal_environment)
@sentry_sdk.trace
@ -117,11 +120,11 @@ with downloader_image.imports():
max_containers=config.video_downloader_concurrency,
volumes={
"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment),
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment),
),
},
secrets=[modal.Secret.from_name("tencent-cloud-secret", environment_name=config.environment)])
secrets=[modal.Secret.from_name("tencent-cloud-secret", environment_name=config.modal_environment)])
@modal.concurrent(max_inputs=10)
async def cache_submit(media: MediaSource, sentry_trace: SentryTransactionInfo) -> MediaSource:
def vod_init():
@ -141,7 +144,7 @@ with downloader_image.imports():
media_info = response.MediaInfoSet[0].BasicInfo
logger.info(f"VOD info = {media_info}")
file_extension = media_info.Type
cache_dir = f"/{config.s3_mount_dir}/{media.protocol.value}/{media.endpoint}/{media.bucket}"
cache_dir = f"/{config.S3_mount_dir}/{media.protocol.value}/{media.endpoint}/{media.bucket}"
cache_file = media.path if '.' in media.path else f"{media.path}.{file_extension}"
return (cache_dir, cache_file, media_info.MediaUrl)
else:
@ -280,7 +283,7 @@ with downloader_image.imports():
process_span.set_status("failed")
case MediaProtocol.http:
try:
cache_filepath = f"{config.s3_mount_dir}/{media.cache_filepath}"
cache_filepath = f"{config.S3_mount_dir}/{media.cache_filepath}"
download_large_file(url=media.__str__(), output_path=cache_filepath)
except Exception as e:
logger.exception(e)
@ -290,8 +293,8 @@ with downloader_image.imports():
process_span.set_status("failed")
case MediaProtocol.s3:
# 本地挂载缓存
if media.protocol == MediaProtocol.s3 and media.endpoint == config.s3_region and media.bucket == config.s3_bucket_name:
volume_cache_path = f"{config.s3_mount_dir}/{media.cache_filepath}"
if media.protocol == MediaProtocol.s3 and media.endpoint == config.S3_region and media.bucket == config.S3_bucket_name:
volume_cache_path = f"{config.S3_mount_dir}/{media.cache_filepath}"
else:
logger.error("protocol not yet supported")
case _:
@ -310,8 +313,8 @@ with downloader_image.imports():
max_containers=config.video_downloader_concurrency,
volumes={
"/mntS3": modal.CloudBucketMount(
bucket_name=config.s3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.environment),
bucket_name=config.S3_bucket_name,
secret=modal.Secret.from_name("aws-s3-secret", environment_name=config.modal_environment),
),
})
@modal.concurrent(max_inputs=10)

2
src/cluster/web.py
View File

@ -1,9 +1,11 @@
import modal
from dotenv import dotenv_values
fastapi_image = (
modal.Image
.debian_slim(python_version="3.11")
.pip_install_from_pyproject("pyproject.toml")
.env(dotenv_values("../../.runtime.env"))
.add_local_python_source('cluster')
.add_local_python_source('BowongModalFunctions')
)

132
uv.lock
View File

@ -166,6 +166,67 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f9/92/2c522e277c95d35b4b83bff6a3839875d91b0d835a93545828a7046013c4/botocore-1.38.10-py3-none-any.whl", hash = "sha256:5244454bb9e8fbb6510145d1554e82fd243e8583507d83077ecf4f8efb66cb46", size = 13539530 },
]
[[package]]
name = "bowongmodalfunctions"
source = { editable = "." }
dependencies = [
{ name = "backoff" },
{ name = "boto3" },
{ name = "cos-python-sdk-v5" },
{ name = "crcmod" },
{ name = "fastapi", extra = ["standard"] },
{ name = "httpx" },
{ name = "loguru" },
{ name = "modal" },
{ name = "noisereduce" },
{ name = "pedalboard" },
{ name = "psutil" },
{ name = "pydantic" },
{ name = "pydantic-settings" },
{ name = "pyloudnorm" },
{ name = "python-dotenv" },
{ name = "python-ffmpeg" },
{ name = "scalar-fastapi" },
{ name = "sentry-sdk", extra = ["loguru"] },
{ name = "soundfile" },
{ name = "tencentcloud-sdk-python-common" },
{ name = "tencentcloud-sdk-python-vod" },
{ name = "webvtt-py" },
]
[package.optional-dependencies]
cli = [
{ name = "twine" },
]
[package.metadata]
requires-dist = [
{ name = "backoff", specifier = ">=2.2.1" },
{ name = "boto3", specifier = ">=1.37.37" },
{ name = "cos-python-sdk-v5", specifier = ">=1.9.36" },
{ name = "crcmod", specifier = ">=1.7" },
{ name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" },
{ name = "httpx", specifier = ">=0.28.1" },
{ name = "loguru", specifier = ">=0.7.3" },
{ name = "modal", specifier = ">=0.76.3" },
{ name = "noisereduce", specifier = ">=3.0.3" },
{ name = "pedalboard", specifier = "==0.9.2" },
{ name = "psutil", specifier = ">=7.0.0" },
{ name = "pydantic", specifier = ">=2.11.3" },
{ name = "pydantic-settings", specifier = ">=2.9.1" },
{ name = "pyloudnorm", specifier = ">=0.1.1" },
{ name = "python-dotenv", specifier = ">=1.1.0" },
{ name = "python-ffmpeg", specifier = ">=2.0.12" },
{ name = "scalar-fastapi", specifier = ">=1.0.3" },
{ name = "sentry-sdk", extras = ["loguru"], specifier = ">=2.26.1" },
{ name = "soundfile", specifier = ">=0.13.1" },
{ name = "tencentcloud-sdk-python-common", specifier = ">=3.0.1363" },
{ name = "tencentcloud-sdk-python-vod", specifier = ">=3.0.1363" },
{ name = "twine", marker = "extra == 'cli'", specifier = ">=6.1.0" },
{ name = "webvtt-py", specifier = ">=0.5.1" },
]
provides-extras = ["cli"]
[[package]]
name = "certifi"
version = "2025.4.26"
@ -1023,7 +1084,7 @@ wheels = [
[[package]]
name = "modal"
version = "0.74.57"
version = "0.77.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "aiohttp" },
@ -1040,68 +1101,9 @@ dependencies = [
{ name = "typing-extensions" },
{ name = "watchfiles" },
]
sdist = { url = "https://files.pythonhosted.org/packages/93/6e/ac879223fbe05b0dae9eb8bf854d5b2e635638929b71a6332cc291972a6f/modal-0.74.57.tar.gz", hash = "sha256:5d598eee638da862a574a433ba55b83ed1ab617e802982c6ff589913307b7a77", size = 505567 }
sdist = { url = "https://files.pythonhosted.org/packages/32/06/acd53f8fb8d4f0f7a1d5a5bfdb962fcbbe917ed6f703d2a6dfdbfb490c34/modal-0.77.0.tar.gz", hash = "sha256:0a3c2b438a5d26c4b2a1100a7cecdd47e09f715e2a6e0de2a50aad235ddc92c3", size = 508716 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/2d/8d/92b981506e3ce3102eb36f2cfab2e980ed0930bf6b39023d9a4896cb460a/modal-0.74.57-py3-none-any.whl", hash = "sha256:a576c2aa9db6c47e8ce3e36f5159d77e4fa807676d9ec22f0a92c9d92ad2bf74", size = 573992 },
]
[[package]]
name = "modaldeploy"
version = "0.2.0"
source = { virtual = "." }
dependencies = [
{ name = "backoff" },
{ name = "boto3" },
{ name = "cos-python-sdk-v5" },
{ name = "crcmod" },
{ name = "fastapi", extra = ["standard"] },
{ name = "httpx" },
{ name = "loguru" },
{ name = "modal" },
{ name = "noisereduce" },
{ name = "pedalboard" },
{ name = "psutil" },
{ name = "pydantic" },
{ name = "pydantic-settings" },
{ name = "pyloudnorm" },
{ name = "python-ffmpeg" },
{ name = "requests" },
{ name = "scalar-fastapi" },
{ name = "sentry-sdk", extra = ["loguru"] },
{ name = "soundfile" },
{ name = "tencentcloud-sdk-python-common" },
{ name = "tencentcloud-sdk-python-vod" },
{ name = "tqdm" },
{ name = "twine" },
{ name = "webvtt-py" },
]
[package.metadata]
requires-dist = [
{ name = "backoff", specifier = ">=2.2.1" },
{ name = "boto3", specifier = ">=1.37.37" },
{ name = "cos-python-sdk-v5", specifier = ">=1.9.36" },
{ name = "crcmod", specifier = ">=1.7" },
{ name = "fastapi", extras = ["standard"], specifier = ">=0.115.12" },
{ name = "httpx", specifier = ">=0.28.1" },
{ name = "loguru", specifier = ">=0.7.3" },
{ name = "modal", specifier = ">=0.74.57" },
{ name = "noisereduce", specifier = ">=3.0.3" },
{ name = "pedalboard", specifier = "==0.9.2" },
{ name = "psutil", specifier = ">=7.0.0" },
{ name = "pydantic", specifier = ">=2.11.3" },
{ name = "pydantic-settings", specifier = ">=2.9.1" },
{ name = "pyloudnorm", specifier = ">=0.1.1" },
{ name = "python-ffmpeg", specifier = ">=2.0.12" },
{ name = "requests", specifier = ">=2.32.3" },
{ name = "scalar-fastapi", specifier = ">=1.0.3" },
{ name = "sentry-sdk", extras = ["loguru"], specifier = ">=2.26.1" },
{ name = "soundfile", specifier = ">=0.13.1" },
{ name = "tencentcloud-sdk-python-common", specifier = ">=3.0.1363" },
{ name = "tencentcloud-sdk-python-vod", specifier = ">=3.0.1363" },
{ name = "tqdm", specifier = ">=4.67.1" },
{ name = "twine", specifier = ">=6.1.0" },
{ name = "webvtt-py", specifier = ">=0.5.1" },
{ url = "https://files.pythonhosted.org/packages/db/a2/06b274bbec3501de530e8be5da02e62258f568201200bcb115976ae3b68e/modal-0.77.0-py3-none-any.whl", hash = "sha256:f646d47e9950b929a03918124c4b12663fdc146287af3c0bd96de3ce23776399", size = 576782 },
]
[[package]]
@ -1981,15 +1983,15 @@ wheels = [
[[package]]
name = "synchronicity"
version = "0.9.11"
version = "0.9.12"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "sigtools" },
{ name = "typing-extensions" },
]
sdist = { url = "https://files.pythonhosted.org/packages/b5/52/f34a9ab6d514e0808d0f572affb360411d596b3439107318c00889277dd6/synchronicity-0.9.11.tar.gz", hash = "sha256:cb5dbbcb43d637e516ae50db05a776da51a705d1e1a9c0e301f6049afc3c2cae", size = 50323 }
sdist = { url = "https://files.pythonhosted.org/packages/ed/21/82c2b51f901452642cb2ed69ab872ef5eacbf98dbd0b661d3673a76d79f5/synchronicity-0.9.12.tar.gz", hash = "sha256:977f3ed8f6e35de4d1a3f0aeee4937143ba8d913f531d33e8df7c539b2792fb8", size = 50441 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f2/d5/7675cd9b8e18f05b9ea261acad5d197fcb8027d2a65b1a750427ec084593/synchronicity-0.9.11-py3-none-any.whl", hash = "sha256:231129654d2f56b1aa148e85ebd8545231be135771f6d2196d414175b1594ef6", size = 36827 },
{ url = "https://files.pythonhosted.org/packages/ab/e3/730a67bee380f5638731cd563e8f5ad3a2d480ee788b767698e83eb2290f/synchronicity-0.9.12-py3-none-any.whl", hash = "sha256:b006f57bd216d55e578316096a11b6dc16016d6b48e2766bcffabe40c88f9793", size = 36820 },
]
[[package]]