From 976f213cbbd1d29f2cafffcdbf4ad22e5828b6fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E5=BA=B7=E5=AE=87=E4=BD=B3?= Date: Tue, 24 Jun 2025 18:34:23 +0800 Subject: [PATCH] =?UTF-8?q?=E5=90=88=E5=B9=B6=E5=88=86=E6=94=AF?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * PERF 修复prompt的label选择 * Merge branch 'main' into cluster-gemini * PERF Gemini推理增加SystemInstruct * PERF 增加上下文商品信息参数 --------- Merge request URL: https://g-ldyi2063.coding.net/p/dev/d/modalDeploy/git/merge/4867 Co-authored-by: 康宇佳 --- src/BowongModalFunctions/utils/HTTPUtils.py | 9 +++-- src/cluster/video_apps/hls_slice_inference.py | 35 +++++++++++++++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/BowongModalFunctions/utils/HTTPUtils.py b/src/BowongModalFunctions/utils/HTTPUtils.py index 1d9fdfe..3c150c4 100644 --- a/src/BowongModalFunctions/utils/HTTPUtils.py +++ b/src/BowongModalFunctions/utils/HTTPUtils.py @@ -7,7 +7,7 @@ import backoff import httpx import asyncio -from google.genai.types import GenerateContentResponse +from google.genai.types import GenerateContentResponse, ContentUnion from loguru import logger import aiofiles from pathlib import Path @@ -143,6 +143,11 @@ class GoogleAuthUtils: def safety_settings(self) -> Optional[List[types.SafetySetting]]: return self.generation_config.safety_settings + @computed_field(alias="systemInstruction") + @property + def system_instruction(self) -> Optional[ContentUnion]: + return self.generation_config.system_instruction + class GoogleGenaiClient(BaseModel): cloudflare_project_id: str cloudflare_gateway_id: str @@ -160,7 +165,7 @@ class GoogleAuthUtils: timeout: int = 30) -> tuple[dict[Any, Any], int] | tuple[GenerateContentResponse, int]: parameter_model = GoogleAuthUtils.VertexAIRequestModel(contents=contents, generationConfig=config) json_body = parameter_model.model_dump_json(indent=2, exclude_none=True, by_alias=True, - exclude={"generation_config": {"safety_settings"}}) + exclude={"generation_config": ["safety_settings", "system_instruction"]}) logger.info(json_body) url = f"{self.gateway_url}/{model_id}:generateContent" logger.info(f"Authorization : Bearer {self.access_token}") diff --git a/src/cluster/video_apps/hls_slice_inference.py b/src/cluster/video_apps/hls_slice_inference.py index 0770657..18f1f38 100644 --- a/src/cluster/video_apps/hls_slice_inference.py +++ b/src/cluster/video_apps/hls_slice_inference.py @@ -94,11 +94,18 @@ with downloader_image.imports(): return { "temperature": c["temperature"], "top_p": c["top_p"], - "safety_settings": c["safety_settings"] - },c + "safety_settings": c["safety_settings"], + "system_instruction": c["system_instruction"] + },{ + "temperature": c["temperature"], + "top_p": c["top_p"], + "response_mime_type": c["response_mime_type"], + "response_schema": c["response_schema"], + "safety_settings": c["safety_settings"] + } # 动态Prompt, langfuse获取失败使用默认值 - IMAGE_PRODUCT_IDENTIFICATION_PROMPT = langfuse.get_prompt("Gemini自动切条/商品识别", type="text", label="latest") + IMAGE_PRODUCT_IDENTIFICATION_PROMPT = langfuse.get_prompt("Gemini自动切条/商品识别", type="text", label="production") if IMAGE_PRODUCT_IDENTIFICATION_PROMPT: prompt_config = IMAGE_PRODUCT_IDENTIFICATION_PROMPT.config first_stage_generate_config, first_stage_correct_config = split_prompt_config(prompt_config) @@ -179,10 +186,17 @@ with downloader_image.imports(): first_stage_generate_config = { "temperature": first_stage_correct_config["temperature"], "top_p": first_stage_correct_config["top_p"], - "safety_settings": first_stage_correct_config["safety_settings"] + "safety_settings": first_stage_correct_config["safety_settings"], + "system_instruction":{ + "parts": [ + { + "text": "你是商品识别专家，任务是从商品网格图片中精准识别商品、提取特征，输出为用户定义的json格式" + } + ] + } } - VIDEO_TIMELINE_ANALYSIS_PROMPT = langfuse.get_prompt("Gemini自动切条/视频时间点识别", type="text", label="latest") + VIDEO_TIMELINE_ANALYSIS_PROMPT = langfuse.get_prompt("Gemini自动切条/视频时间点识别", type="text", label="production") if VIDEO_TIMELINE_ANALYSIS_PROMPT: prompt_config = VIDEO_TIMELINE_ANALYSIS_PROMPT.config second_stage_generate_config, second_stage_correct_config = split_prompt_config(prompt_config) @@ -277,7 +291,14 @@ with downloader_image.imports(): second_stage_generate_config = { "temperature": second_stage_correct_config["temperature"], "top_p": second_stage_correct_config["top_p"], - "safety_settings": second_stage_correct_config["safety_settings"] + "safety_settings": second_stage_correct_config["safety_settings"], + "system_instruction":{ + "parts": [ + { + "text": "你是直播带货视频识别专家，任务是根据商品特征从视频中高效精准地识别出视频中每个商品出现的时间段以及内容类型，输出为用户定义的json格式" + } + ] + } } @@ -308,7 +329,7 @@ with downloader_image.imports(): types.Part.from_text( text="" "" - "请格式化以下一段非标准json格式字符串为json标准格式 \n{0}" + "格式化以下一段有问题的json, json可能存在格式错误字段错误以及部分缺失，修复后输出，若product字段中存在双引号等需转义字符需要转义 \n{0}" "" "".format( json_like_str