779 lines
29 KiB
Python
779 lines
29 KiB
Python
from fastapi import APIRouter, HTTPException
|
|
from fastapi.responses import HTMLResponse
|
|
import psutil
|
|
import asyncio
|
|
import logging
|
|
from datetime import datetime, timedelta
|
|
from typing import Dict, Any, List
|
|
import json
|
|
|
|
# 设置日志
|
|
logger = logging.getLogger(__name__)
|
|
|
|
from workflow_service.config import settings
|
|
from workflow_service.database.api import (
|
|
get_workflow_runs_recent,
|
|
get_workflow_run_nodes,
|
|
)
|
|
from workflow_service.comfy.comfy_queue import WorkflowQueueManager
|
|
from workflow_service.comfy.comfy_server import server_manager
|
|
|
|
monitor_router = APIRouter(prefix="/monitor", tags=["监控"])
|
|
|
|
# 全局队列管理器实例
|
|
queue_manager = WorkflowQueueManager()
|
|
|
|
|
|
@monitor_router.get("/", response_class=HTMLResponse)
|
|
async def monitor_dashboard():
|
|
"""监控仪表板页面"""
|
|
html_content = """
|
|
<!DOCTYPE html>
|
|
<html lang="zh-CN">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
|
<title>ComfyUI 工作流监控</title>
|
|
<style>
|
|
* {
|
|
margin: 0;
|
|
padding: 0;
|
|
box-sizing: border-box;
|
|
}
|
|
|
|
body {
|
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
|
|
background: linear-gradient(135deg, #f5f7fa 0%, #c3cfe2 100%);
|
|
min-height: 100vh;
|
|
padding: 24px;
|
|
color: #2c3e50;
|
|
line-height: 1.6;
|
|
}
|
|
|
|
.container {
|
|
max-width: 1200px;
|
|
margin: 0 auto;
|
|
}
|
|
|
|
.refresh-btn {
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
color: white;
|
|
border: none;
|
|
padding: 12px 24px;
|
|
border-radius: 8px;
|
|
cursor: pointer;
|
|
font-size: 14px;
|
|
font-weight: 500;
|
|
margin-bottom: 24px;
|
|
box-shadow: 0 4px 15px rgba(102, 126, 234, 0.3);
|
|
transition: all 0.3s ease;
|
|
}
|
|
|
|
.refresh-btn:hover {
|
|
transform: translateY(-2px);
|
|
box-shadow: 0 6px 20px rgba(102, 126, 234, 0.4);
|
|
}
|
|
|
|
.stats-grid {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
gap: 20px;
|
|
margin-bottom: 24px;
|
|
}
|
|
|
|
.stat-card {
|
|
background: rgba(255, 255, 255, 0.95);
|
|
padding: 24px;
|
|
border-radius: 16px;
|
|
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
|
|
text-align: center;
|
|
backdrop-filter: blur(10px);
|
|
border: 1px solid rgba(255, 255, 255, 0.2);
|
|
transition: all 0.3s ease;
|
|
}
|
|
|
|
.stat-card:hover {
|
|
transform: translateY(-4px);
|
|
box-shadow: 0 12px 40px rgba(0, 0, 0, 0.15);
|
|
}
|
|
|
|
.stat-number {
|
|
font-size: 2.2em;
|
|
font-weight: 700;
|
|
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
|
-webkit-background-clip: text;
|
|
-webkit-text-fill-color: transparent;
|
|
background-clip: text;
|
|
margin: 8px 0;
|
|
}
|
|
|
|
.stat-label {
|
|
color: #7f8c8d;
|
|
font-size: 0.9em;
|
|
font-weight: 500;
|
|
text-transform: uppercase;
|
|
letter-spacing: 0.5px;
|
|
}
|
|
|
|
.section {
|
|
background: rgba(255, 255, 255, 0.95);
|
|
padding: 24px;
|
|
border-radius: 16px;
|
|
box-shadow: 0 8px 32px rgba(0, 0, 0, 0.1);
|
|
margin-bottom: 24px;
|
|
backdrop-filter: blur(10px);
|
|
border: 1px solid rgba(255, 255, 255, 0.2);
|
|
}
|
|
|
|
.section h3 {
|
|
margin-bottom: 20px;
|
|
color: #2c3e50;
|
|
font-size: 1.3em;
|
|
font-weight: 600;
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 8px;
|
|
}
|
|
|
|
.server-status {
|
|
display: grid;
|
|
grid-template-columns: repeat(auto-fit, minmax(280px, 1fr));
|
|
gap: 16px;
|
|
}
|
|
|
|
.server-card {
|
|
border: 1px solid #e1e8ed;
|
|
border-radius: 8px;
|
|
padding: 12px;
|
|
background: #f8f9fa;
|
|
transition: all 0.3s ease;
|
|
position: relative;
|
|
overflow: hidden;
|
|
}
|
|
|
|
.server-card::before {
|
|
content: '';
|
|
position: absolute;
|
|
top: 0;
|
|
left: 0;
|
|
right: 0;
|
|
height: 4px;
|
|
background: #e1e8ed;
|
|
}
|
|
|
|
.server-card.online {
|
|
border-color: #27ae60;
|
|
background: #f8fff9;
|
|
}
|
|
|
|
.server-card.online::before {
|
|
background: #27ae60;
|
|
}
|
|
|
|
.server-card.offline {
|
|
border-color: #e74c3c;
|
|
background: #fff8f8;
|
|
}
|
|
|
|
.server-card.offline::before {
|
|
background: #e74c3c;
|
|
}
|
|
|
|
.server-header {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 12px;
|
|
margin-bottom: 8px;
|
|
}
|
|
|
|
.server-card h4 {
|
|
color: #2c3e50;
|
|
margin: 0;
|
|
font-size: 1em;
|
|
font-weight: 600;
|
|
white-space: nowrap;
|
|
}
|
|
|
|
.server-url {
|
|
color: #3498db;
|
|
font-size: 0.8em;
|
|
font-weight: 500;
|
|
font-family: monospace;
|
|
white-space: nowrap;
|
|
overflow: hidden;
|
|
text-overflow: ellipsis;
|
|
max-width: 200px;
|
|
}
|
|
|
|
.server-info {
|
|
display: flex;
|
|
flex-wrap: wrap;
|
|
gap: 8px;
|
|
align-items: center;
|
|
font-size: 0.8em;
|
|
margin-top: 8px;
|
|
}
|
|
|
|
.info-item {
|
|
display: flex;
|
|
align-items: center;
|
|
gap: 4px;
|
|
white-space: nowrap;
|
|
}
|
|
|
|
.info-separator {
|
|
color: #e1e8ed;
|
|
margin: 0 4px;
|
|
}
|
|
|
|
.status-icon {
|
|
font-size: 0.9em;
|
|
}
|
|
|
|
.status-text {
|
|
font-weight: 600;
|
|
color: #2c3e50;
|
|
}
|
|
|
|
.info-label {
|
|
color: #7f8c8d;
|
|
font-weight: 500;
|
|
font-size: 0.75em;
|
|
}
|
|
|
|
.info-value {
|
|
color: #2c3e50;
|
|
font-weight: 600;
|
|
font-size: 0.75em;
|
|
}
|
|
|
|
.info-value:has(span[style*="color: #e74c3c"]) {
|
|
color: #e74c3c;
|
|
}
|
|
|
|
.info-value:has(span[style*="color: #f39c12"]) {
|
|
color: #f39c12;
|
|
}
|
|
|
|
.info-value:has(span[style*="color: #27ae60"]) {
|
|
color: #27ae60;
|
|
}
|
|
|
|
.status-indicator {
|
|
display: inline-block;
|
|
width: 8px;
|
|
height: 8px;
|
|
border-radius: 50%;
|
|
margin-right: 8px;
|
|
}
|
|
|
|
.status-online { background-color: #27ae60; }
|
|
.status-offline { background-color: #e74c3c; }
|
|
|
|
.task-list {
|
|
max-height: 500px;
|
|
overflow-y: auto;
|
|
}
|
|
|
|
.task-item {
|
|
border: 1px solid #e1e8ed;
|
|
border-radius: 8px;
|
|
padding: 16px;
|
|
margin: 8px 0;
|
|
background: #f8f9fa;
|
|
transition: all 0.3s ease;
|
|
}
|
|
|
|
.task-item:hover {
|
|
background: #ffffff;
|
|
box-shadow: 0 4px 12px rgba(0, 0, 0, 0.1);
|
|
}
|
|
|
|
.task-header {
|
|
display: flex;
|
|
justify-content: space-between;
|
|
align-items: center;
|
|
margin-bottom: 8px;
|
|
}
|
|
|
|
.task-name {
|
|
font-weight: 600;
|
|
color: #2c3e50;
|
|
}
|
|
|
|
.task-status {
|
|
padding: 4px 12px;
|
|
border-radius: 20px;
|
|
font-size: 0.8em;
|
|
font-weight: 600;
|
|
color: white;
|
|
text-transform: uppercase;
|
|
letter-spacing: 0.5px;
|
|
}
|
|
|
|
.status-pending { background: #f39c12; }
|
|
.status-running { background: #3498db; }
|
|
.status-completed { background: #27ae60; }
|
|
.status-failed { background: #e74c3c; }
|
|
|
|
.task-meta {
|
|
color: #7f8c8d;
|
|
font-size: 0.85em;
|
|
}
|
|
|
|
.error-message {
|
|
color: #e74c3c;
|
|
font-size: 0.85em;
|
|
margin-top: 8px;
|
|
padding: 8px;
|
|
background: #fff5f5;
|
|
border-radius: 6px;
|
|
border-left: 3px solid #e74c3c;
|
|
}
|
|
|
|
@media (max-width: 768px) {
|
|
body { padding: 16px; }
|
|
.stats-grid { grid-template-columns: repeat(2, 1fr); }
|
|
.server-status { grid-template-columns: 1fr; }
|
|
.server-info { grid-template-columns: 1fr; }
|
|
}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<div class="container">
|
|
<button class="refresh-btn" onclick="refreshData()">🔄 刷新数据</button>
|
|
|
|
<div class="stats-grid">
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="running-tasks">--</div>
|
|
<div class="stat-label">运行中任务</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="pending-tasks">--</div>
|
|
<div class="stat-label">等待中任务</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="completed-tasks">--</div>
|
|
<div class="stat-label">已完成任务</div>
|
|
</div>
|
|
<div class="stat-card">
|
|
<div class="stat-number" id="total-tasks">--</div>
|
|
<div class="stat-label">24小时总任务</div>
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h3>🖥️ 服务器状态</h3>
|
|
<div class="server-status" id="server-status">
|
|
<!-- 服务器状态将在这里动态加载 -->
|
|
</div>
|
|
</div>
|
|
|
|
<div class="section">
|
|
<h3>📋 最近任务</h3>
|
|
<div class="task-list" id="recent-tasks">
|
|
<!-- 最近任务将在这里动态加载 -->
|
|
</div>
|
|
</div>
|
|
</div>
|
|
|
|
<script>
|
|
async function refreshData() {
|
|
try {
|
|
// 获取任务状态
|
|
const taskStats = await fetch('/monitor/task-stats').then(r => r.json());
|
|
document.getElementById('running-tasks').textContent = taskStats.running_tasks;
|
|
document.getElementById('pending-tasks').textContent = taskStats.pending_tasks;
|
|
document.getElementById('completed-tasks').textContent = taskStats.completed_tasks;
|
|
document.getElementById('total-tasks').textContent = taskStats.total_tasks_24h;
|
|
|
|
// 获取服务器状态
|
|
const serverStatus = await fetch('/monitor/server-status').then(r => r.json());
|
|
updateServerStatus(serverStatus);
|
|
|
|
// 获取最近任务
|
|
const recentTasks = await fetch('/monitor/recent-tasks').then(r => r.json());
|
|
updateRecentTasks(recentTasks);
|
|
|
|
} catch (error) {
|
|
console.error('刷新数据失败:', error);
|
|
}
|
|
}
|
|
|
|
function updateServerStatus(servers) {
|
|
const container = document.getElementById('server-status');
|
|
container.innerHTML = '';
|
|
|
|
if (servers.length === 0) {
|
|
container.innerHTML = '<p style="text-align: center; color: #7f8c8d; grid-column: 1/-1;">暂无服务器信息</p>';
|
|
return;
|
|
}
|
|
|
|
servers.forEach(server => {
|
|
const statusClass = server.status === 'online' ? 'online' : 'offline';
|
|
const statusColor = server.status === 'online' ? 'status-online' : 'status-offline';
|
|
|
|
// 格式化时间
|
|
const formatTime = (timeStr) => {
|
|
if (!timeStr) return 'N/A';
|
|
try {
|
|
const date = new Date(timeStr);
|
|
const now = new Date();
|
|
const diff = now - date;
|
|
|
|
if (diff < 60000) return '刚刚';
|
|
if (diff < 3600000) return Math.floor(diff / 60000) + '分钟前';
|
|
if (diff < 86400000) return Math.floor(diff / 3600000) + '小时前';
|
|
return date.toLocaleDateString();
|
|
} catch {
|
|
return timeStr;
|
|
}
|
|
};
|
|
|
|
// 格式化心跳时间，突出显示状态
|
|
const formatHeartbeat = (timeStr) => {
|
|
if (!timeStr) return 'N/A';
|
|
try {
|
|
const date = new Date(timeStr);
|
|
const now = new Date();
|
|
const diff = now - date;
|
|
|
|
if (diff > 86400000) return '⚠️ 心跳超时'; // 超过1天
|
|
if (diff > 3600000) return '⚠️ 心跳延迟'; // 超过1小时
|
|
if (diff < 60000) return '✅ 刚刚';
|
|
if (diff < 3600000) return `✅ ${Math.floor(diff / 60000)}分钟前`;
|
|
return `✅ ${Math.floor(diff / 3600000)}小时前`;
|
|
} catch {
|
|
return timeStr;
|
|
}
|
|
};
|
|
|
|
container.innerHTML += `
|
|
<div class="server-card ${statusClass}">
|
|
<div class="server-header">
|
|
<h4>${server.name || '未命名'}</h4>
|
|
<span class="server-url">📍 ${server.http_url}</span>
|
|
</div>
|
|
<div class="server-info">
|
|
<div class="info-item">
|
|
<span class="status-icon">${server.status === 'online' ? '🟢' : '🔴'}</span>
|
|
<span class="status-text">${server.status === 'online' ? '在线' : '离线'}</span>
|
|
</div>
|
|
<span class="info-separator">|</span>
|
|
<div class="info-item">
|
|
<span class="status-text">任务: ${server.current_tasks || 0}/${server.max_concurrent_tasks || 1}</span>
|
|
</div>
|
|
<span class="info-separator">|</span>
|
|
<div class="info-item">
|
|
<span class="status-text">心跳: ${formatHeartbeat(server.last_heartbeat)}</span>
|
|
</div>
|
|
<span class="info-separator">|</span>
|
|
<div class="info-item">
|
|
<span class="status-text">检查: ${formatTime(server.last_health_check)}</span>
|
|
</div>
|
|
</div>
|
|
${server.error ? `<div class="error-message">${server.error}</div>` : ''}
|
|
</div>
|
|
`;
|
|
});
|
|
}
|
|
|
|
function updateRecentTasks(tasks) {
|
|
const container = document.getElementById('recent-tasks');
|
|
container.innerHTML = '';
|
|
|
|
if (tasks.length === 0) {
|
|
container.innerHTML = '<p style="text-align: center; color: #7f8c8d;">暂无任务记录</p>';
|
|
return;
|
|
}
|
|
|
|
tasks.forEach(task => {
|
|
const statusClass = 'status-' + task.status.toLowerCase();
|
|
const statusText = {
|
|
'pending': '等待中',
|
|
'running': '运行中',
|
|
'completed': '已完成',
|
|
'failed': '失败'
|
|
}[task.status] || task.status;
|
|
|
|
container.innerHTML += `
|
|
<div class="task-item">
|
|
<div class="task-header">
|
|
<div class="task-name">${task.workflow_name || '未命名工作流'}</div>
|
|
<span class="task-status ${statusClass}">${statusText}</span>
|
|
</div>
|
|
<div class="task-meta">
|
|
<div>ID: ${task.id}</div>
|
|
<div>创建时间: ${new Date(task.created_at).toLocaleString()}</div>
|
|
</div>
|
|
</div>
|
|
`;
|
|
});
|
|
}
|
|
|
|
// 页面加载时自动刷新数据
|
|
document.addEventListener('DOMContentLoaded', refreshData);
|
|
|
|
// 每30秒自动刷新一次
|
|
setInterval(refreshData, 30000);
|
|
</script>
|
|
</body>
|
|
</html>
|
|
"""
|
|
return HTMLResponse(content=html_content)
|
|
|
|
|
|
@monitor_router.get("/system-stats")
|
|
async def get_system_stats() -> Dict[str, Any]:
|
|
"""获取系统统计信息"""
|
|
try:
|
|
cpu_percent = psutil.cpu_percent(interval=1)
|
|
memory = psutil.virtual_memory()
|
|
disk = psutil.disk_usage("/")
|
|
|
|
return {
|
|
"cpu_percent": round(cpu_percent, 1),
|
|
"memory_percent": round(memory.percent, 1),
|
|
"memory_used_gb": round(memory.used / (1024**3), 2),
|
|
"memory_total_gb": round(memory.total / (1024**3), 2),
|
|
"disk_percent": round(disk.percent, 1),
|
|
"disk_used_gb": round(disk.used / (1024**3), 2),
|
|
"disk_total_gb": round(disk.total / (1024**3), 2),
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"获取系统统计信息失败: {str(e)}")
|
|
|
|
|
|
@monitor_router.get("/task-stats")
|
|
async def get_task_stats() -> Dict[str, Any]:
|
|
"""获取任务统计信息"""
|
|
try:
|
|
# 获取最近24小时的任务统计
|
|
end_time = datetime.now()
|
|
start_time = end_time - timedelta(hours=24)
|
|
|
|
recent_runs = await get_workflow_runs_recent(start_time, end_time)
|
|
|
|
# 统计各种状态的任务数量
|
|
status_counts = {}
|
|
for run in recent_runs:
|
|
status = run.get("status", "unknown")
|
|
status_counts[status] = status_counts.get(status, 0) + 1
|
|
|
|
return {
|
|
"running_tasks": status_counts.get("running", 0),
|
|
"pending_tasks": status_counts.get("pending", 0),
|
|
"completed_tasks": status_counts.get("completed", 0),
|
|
"failed_tasks": status_counts.get("failed", 0),
|
|
"total_tasks_24h": len(recent_runs),
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"获取任务统计信息失败: {str(e)}")
|
|
|
|
|
|
@monitor_router.get("/server-status")
|
|
async def get_server_status() -> List[Dict[str, Any]]:
|
|
"""获取服务器状态信息"""
|
|
try:
|
|
import aiohttp
|
|
import asyncio
|
|
|
|
# 从 ComfyUIServerManager 获取所有注册的服务器
|
|
all_servers = await server_manager.get_all_servers()
|
|
|
|
if not all_servers:
|
|
logger.info("当前没有动态注册的服务器")
|
|
return []
|
|
|
|
async def check_server_status(server):
|
|
try:
|
|
timeout = aiohttp.ClientTimeout(total=5)
|
|
async with aiohttp.ClientSession(timeout=timeout) as session:
|
|
async with session.get(
|
|
f"{server.http_url}/system_stats"
|
|
) as response:
|
|
if response.status == 200:
|
|
return {
|
|
"name": server.name,
|
|
"http_url": server.http_url,
|
|
"ws_url": server.ws_url,
|
|
"status": "online",
|
|
"response_time": response.headers.get(
|
|
"X-Response-Time", "N/A"
|
|
),
|
|
"current_tasks": getattr(server, "current_tasks", 0),
|
|
"max_concurrent_tasks": getattr(
|
|
server, "max_concurrent_tasks", 1
|
|
),
|
|
"last_heartbeat": getattr(
|
|
server, "last_heartbeat", None
|
|
),
|
|
"last_health_check": getattr(
|
|
server, "last_health_check", None
|
|
),
|
|
}
|
|
else:
|
|
return {
|
|
"name": server.name,
|
|
"http_url": server.http_url,
|
|
"ws_url": server.ws_url,
|
|
"status": "offline",
|
|
"error": f"HTTP {response.status}",
|
|
"current_tasks": getattr(server, "current_tasks", 0),
|
|
"max_concurrent_tasks": getattr(
|
|
server, "max_concurrent_tasks", 1
|
|
),
|
|
"last_heartbeat": getattr(
|
|
server, "last_heartbeat", None
|
|
),
|
|
"last_health_check": getattr(
|
|
server, "last_health_check", None
|
|
),
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"name": server.name,
|
|
"http_url": server.http_url,
|
|
"ws_url": server.ws_url,
|
|
"status": "offline",
|
|
"error": str(e),
|
|
"current_tasks": getattr(server, "current_tasks", 0),
|
|
"max_concurrent_tasks": getattr(server, "max_concurrent_tasks", 1),
|
|
"last_heartbeat": getattr(server, "last_heartbeat", None),
|
|
"last_health_check": getattr(server, "last_health_check", None),
|
|
}
|
|
|
|
# 并发检查所有服务器状态
|
|
tasks = [check_server_status(server) for server in all_servers]
|
|
results = await asyncio.gather(*tasks, return_exceptions=True)
|
|
|
|
# 过滤掉异常结果
|
|
valid_results = []
|
|
for result in results:
|
|
if isinstance(result, dict):
|
|
valid_results.append(result)
|
|
|
|
return valid_results
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"获取服务器状态失败: {str(e)}")
|
|
|
|
|
|
@monitor_router.get("/recent-tasks")
|
|
async def get_recent_tasks(limit: int = 10) -> List[Dict[str, Any]]:
|
|
"""获取最近的任务列表"""
|
|
try:
|
|
end_time = datetime.now()
|
|
start_time = end_time - timedelta(hours=24)
|
|
|
|
recent_runs = await get_workflow_runs_recent(start_time, end_time)
|
|
|
|
# 限制返回数量
|
|
limited_runs = recent_runs[:limit]
|
|
|
|
# 格式化返回数据
|
|
formatted_runs = []
|
|
for run in limited_runs:
|
|
formatted_runs.append(
|
|
{
|
|
"id": run.get("id"), # 使用数据库中的id字段
|
|
"workflow_name": run.get("workflow_name"),
|
|
"status": run.get("status", "unknown"),
|
|
"created_at": run.get("created_at"),
|
|
"updated_at": run.get("updated_at"),
|
|
"api_spec": run.get("api_spec"),
|
|
}
|
|
)
|
|
|
|
return formatted_runs
|
|
|
|
except Exception as e:
|
|
raise HTTPException(status_code=500, detail=f"获取最近任务失败: {str(e)}")
|
|
|
|
|
|
@monitor_router.get("/health")
|
|
async def health_check() -> Dict[str, Any]:
|
|
"""健康检查端点"""
|
|
try:
|
|
# 检查数据库连接
|
|
from workflow_service.database.connection import get_db
|
|
|
|
db = get_db()
|
|
|
|
# 从 ComfyUIServerManager 获取服务器信息
|
|
all_servers = await server_manager.get_all_servers()
|
|
dynamic_servers_count = len(all_servers)
|
|
|
|
# 检查服务器健康状态
|
|
online_servers = [s for s in all_servers if s.status == "online"]
|
|
offline_servers = [s for s in all_servers if s.status != "online"]
|
|
|
|
return {
|
|
"status": "healthy",
|
|
"timestamp": datetime.now().isoformat(),
|
|
"version": "1.0.0",
|
|
"database": "connected",
|
|
"servers": {
|
|
"total_dynamic": dynamic_servers_count,
|
|
"online": len(online_servers),
|
|
"offline": len(offline_servers),
|
|
},
|
|
"uptime": "N/A", # 可以添加启动时间跟踪
|
|
}
|
|
except Exception as e:
|
|
return {
|
|
"status": "unhealthy",
|
|
"timestamp": datetime.now().isoformat(),
|
|
"error": str(e),
|
|
}
|
|
|
|
|
|
@monitor_router.get("/queue/status")
|
|
async def get_queue_status():
|
|
"""获取队列状态"""
|
|
try:
|
|
status = await queue_manager.get_queue_status()
|
|
return {
|
|
"success": True,
|
|
"data": status,
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"获取队列状态失败: {e}")
|
|
raise HTTPException(status_code=500, detail=f"获取队列状态失败: {str(e)}")
|
|
|
|
|
|
@monitor_router.post("/queue/trigger")
|
|
async def trigger_queue_processing():
|
|
"""手动触发队列处理"""
|
|
try:
|
|
await queue_manager.trigger_queue_processing()
|
|
return {
|
|
"success": True,
|
|
"message": "队列处理已手动触发",
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"手动触发队列处理失败: {e}")
|
|
raise HTTPException(status_code=500, detail=f"手动触发队列处理失败: {str(e)}")
|
|
|
|
|
|
@monitor_router.post("/queue/interval")
|
|
async def set_monitor_interval(interval: int):
|
|
"""设置队列监控间隔"""
|
|
try:
|
|
if interval < 1:
|
|
raise HTTPException(status_code=400, detail="监控间隔不能小于1秒")
|
|
|
|
await queue_manager.set_monitor_interval(interval)
|
|
return {
|
|
"success": True,
|
|
"message": f"队列监控间隔已设置为 {interval} 秒",
|
|
"timestamp": datetime.now().isoformat(),
|
|
}
|
|
except Exception as e:
|
|
logger.error(f"设置监控间隔失败: {e}")
|
|
raise HTTPException(status_code=500, detail=f"设置监控间隔失败: {str(e)}")
|