import json import uuid from typing import Dict, List, Any, Optional, Union from datetime import datetime from python_core.kv import kv from python_core.utils.logger import setup_logger logger = setup_logger(__name__) class Db: """ 基于KV存储的数据库类 提供基础的CRUD操作、索引、查询等功能 """ def __init__(self, key: str): """ 初始化数据库 Args: key: 数据库的唯一标识符,用作KV存储的前缀 """ self.kv = kv self.key = key self.table_prefix = f"db:{key}:table:" self.index_prefix = f"db:{key}:index:" self.meta_prefix = f"db:{key}:meta:" # 初始化数据库元数据 self._init_metadata() def _init_metadata(self): """初始化数据库元数据""" try: meta_key = f"{self.meta_prefix}info" existing_meta = self.kv.get(meta_key) if not existing_meta: # 创建新的数据库元数据 metadata = { "created_at": datetime.now().isoformat(), "version": "1.0.0", "tables": [], "indexes": {} } self.kv.set(meta_key, json.dumps(metadata)) logger.info(f"Initialized new database: {self.key}") else: logger.info(f"Connected to existing database: {self.key}") except Exception as e: logger.error(f"Failed to initialize database metadata: {e}") raise e def _get_metadata(self) -> Dict[str, Any]: """获取数据库元数据""" try: meta_key = f"{self.meta_prefix}info" logger.debug(f"Getting metadata with key: {meta_key}") meta_str = self.kv.get(meta_key) logger.debug(f"Raw metadata string: {meta_str}") if meta_str: parsed = json.loads(meta_str) logger.debug(f"Parsed metadata: {parsed}") return parsed logger.debug("No metadata found, returning empty dict") return {} except Exception as e: logger.error(f"Failed to get database metadata: {e}") import traceback traceback.print_exc() return {} def _update_metadata(self, metadata: Dict[str, Any]): """更新数据库元数据""" try: meta_key = f"{self.meta_prefix}info" logger.debug(f"Updating metadata with key: {meta_key}") logger.debug(f"Metadata content: {metadata}") result = self.kv.set(meta_key, json.dumps(metadata)) logger.debug(f"Update result: {result}") # 添加短暂延迟确保KV同步 import time time.sleep(0.1) except Exception as e: logger.error(f"Failed to update database metadata: {e}") raise e def create_table(self, table_name: str, schema: Dict[str, str] = None) -> bool: """ 创建表 Args: table_name: 表名 schema: 表结构定义 (字段名: 类型) Returns: 创建成功返回True """ try: metadata = self._get_metadata() if table_name in metadata.get("tables", []): logger.warning(f"Table '{table_name}' already exists") return False # 创建表元数据 table_meta = { "name": table_name, "created_at": datetime.now().isoformat(), "schema": schema or {}, "record_count": 0, "indexes": [] } # 保存表元数据 table_meta_key = f"{self.meta_prefix}table:{table_name}" self.kv.set(table_meta_key, json.dumps(table_meta)) # 更新数据库元数据 metadata.setdefault("tables", []).append(table_name) self._update_metadata(metadata) logger.info(f"Created table: {table_name}") return True except Exception as e: logger.error(f"Failed to create table '{table_name}': {e}") raise e def drop_table(self, table_name: str) -> bool: """ 删除表 Args: table_name: 表名 Returns: 删除成功返回True """ try: metadata = self._get_metadata() if table_name not in metadata.get("tables", []): logger.warning(f"Table '{table_name}' does not exist") return False # 获取表中所有记录的键 record_keys = self._get_all_record_keys(table_name) # 删除所有记录 if record_keys: self.kv.removes(record_keys) # 删除表元数据 table_meta_key = f"{self.meta_prefix}table:{table_name}" self.kv.remove(table_meta_key) # 删除相关索引 self._drop_table_indexes(table_name) # 更新数据库元数据 metadata["tables"] = [t for t in metadata.get("tables", []) if t != table_name] self._update_metadata(metadata) logger.info(f"Dropped table: {table_name}") return True except Exception as e: logger.error(f"Failed to drop table '{table_name}': {e}") raise e def insert(self, table_name: str, data: Dict[str, Any], record_id: str = None) -> str: """ 插入记录 Args: table_name: 表名 data: 记录数据 record_id: 记录ID,如果不提供则自动生成 Returns: 记录ID """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") # 生成记录ID if not record_id: import uuid record_id = str(uuid.uuid4()) # 添加元数据 record = { "id": record_id, "data": data, "created_at": datetime.now().isoformat(), "updated_at": datetime.now().isoformat() } # 保存记录 record_key = f"{self.table_prefix}{table_name}:record:{record_id}" self.kv.set(record_key, json.dumps(record)) # 更新表统计 self._update_table_stats(table_name, record_count_delta=1) # 更新索引 self._update_indexes(table_name, record_id, data) logger.debug(f"Inserted record {record_id} into table {table_name}") return record_id except Exception as e: logger.error(f"Failed to insert record into table '{table_name}': {e}") raise e def get(self, table_name: str, record_id: str) -> Optional[Dict[str, Any]]: """ 获取单条记录 Args: table_name: 表名 record_id: 记录ID Returns: 记录数据,如果不存在返回None """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") record_key = f"{self.table_prefix}{table_name}:record:{record_id}" record_str = self.kv.get(record_key) if record_str: record = json.loads(record_str) return record return None except Exception as e: logger.error(f"Failed to get record {record_id} from table '{table_name}': {e}") raise e def update(self, table_name: str, record_id: str, data: Dict[str, Any]) -> bool: """ 更新记录 Args: table_name: 表名 record_id: 记录ID data: 新的数据 Returns: 更新成功返回True """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") # 获取现有记录 existing_record = self.get(table_name, record_id) if not existing_record: logger.warning(f"Record {record_id} not found in table {table_name}") return False # 更新记录 updated_record = { "id": record_id, "data": data, "created_at": existing_record.get("created_at"), "updated_at": datetime.now().isoformat() } # 保存更新后的记录 record_key = f"{self.table_prefix}{table_name}:record:{record_id}" self.kv.set(record_key, json.dumps(updated_record)) # 更新索引 self._update_indexes(table_name, record_id, data, existing_record.get("data")) logger.debug(f"Updated record {record_id} in table {table_name}") return True except Exception as e: logger.error(f"Failed to update record {record_id} in table '{table_name}': {e}") raise e def delete(self, table_name: str, record_id: str) -> bool: """ 删除记录 Args: table_name: 表名 record_id: 记录ID Returns: 删除成功返回True """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") # 获取记录以便清理索引 existing_record = self.get(table_name, record_id) if not existing_record: logger.warning(f"Record {record_id} not found in table {table_name}") return False # 删除记录 record_key = f"{self.table_prefix}{table_name}:record:{record_id}" self.kv.remove(record_key) # 更新表统计 self._update_table_stats(table_name, record_count_delta=-1) # 清理索引 self._remove_from_indexes(table_name, record_id, existing_record.get("data")) logger.debug(f"Deleted record {record_id} from table {table_name}") return True except Exception as e: logger.error(f"Failed to delete record {record_id} from table '{table_name}': {e}") raise e def find_all(self, table_name: str, limit: int = 100) -> List[Dict[str, Any]]: """ 获取表中所有记录 Args: table_name: 表名 limit: 最大返回记录数 Returns: 记录列表 """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") # 获取所有记录键 record_keys = self._get_all_record_keys(table_name, limit) if not record_keys: return [] # 批量获取记录 records_data = self.kv.gets(record_keys) records = [] for key, record_str in records_data.items(): if record_str: try: record = json.loads(record_str) records.append(record) except json.JSONDecodeError: logger.warning(f"Failed to parse record: {key}") return records except Exception as e: logger.error(f"Failed to find all records in table '{table_name}': {e}") raise e def find_by_field(self, table_name: str, field: str, value: Any, limit: int = 100) -> List[Dict[str, Any]]: """ 根据字段值查找记录 Args: table_name: 表名 field: 字段名 value: 字段值 limit: 最大返回记录数 Returns: 匹配的记录列表 """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") # 简单的线性搜索(后续可以优化为索引搜索) all_records = self.find_all(table_name, limit * 2) # 获取更多记录以便过滤 matching_records = [] for record in all_records: if len(matching_records) >= limit: break record_data = record.get("data", {}) if record_data.get(field) == value: matching_records.append(record) return matching_records except Exception as e: logger.error(f"Failed to find records by field in table '{table_name}': {e}") raise e def count(self, table_name: str) -> int: """ 获取表中记录数量 Args: table_name: 表名 Returns: 记录数量 """ try: if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") table_meta = self._get_table_metadata(table_name) return table_meta.get("record_count", 0) except Exception as e: logger.error(f"Failed to count records in table '{table_name}': {e}") raise e # 辅助方法 def _table_exists(self, table_name: str) -> bool: """检查表是否存在""" metadata = self._get_metadata() return table_name in metadata.get("tables", []) def _get_table_metadata(self, table_name: str) -> Dict[str, Any]: """获取表元数据""" table_meta_key = f"{self.meta_prefix}table:{table_name}" meta_str = self.kv.get(table_meta_key) if meta_str: return json.loads(meta_str) return {} def _update_table_stats(self, table_name: str, record_count_delta: int = 0): """更新表统计信息""" try: table_meta = self._get_table_metadata(table_name) table_meta["record_count"] = table_meta.get("record_count", 0) + record_count_delta table_meta_key = f"{self.meta_prefix}table:{table_name}" self.kv.set(table_meta_key, json.dumps(table_meta)) except Exception as e: logger.error(f"Failed to update table stats for '{table_name}': {e}") def _get_all_record_keys(self, table_name: str, limit: int = 1000) -> List[str]: """获取表中所有记录的键""" try: prefix = f"{self.table_prefix}{table_name}:record:" # 使用KV的list_keys功能 result = self.kv.list_keys(prefix=prefix, limit=limit) if result and "result" in result: keys = [] for key_info in result["result"]: if isinstance(key_info, dict): keys.append(key_info.get("name")) else: keys.append(str(key_info)) return keys return [] except Exception as e: logger.error(f"Failed to get record keys for table '{table_name}': {e}") return [] def _update_indexes(self, table_name: str, record_id: str, data: Dict[str, Any], old_data: Dict[str, Any] = None): """更新索引(简单实现,后续可扩展)""" # 这里可以实现索引更新逻辑 # 目前为简单实现,不做复杂索引 pass def _remove_from_indexes(self, table_name: str, record_id: str, data: Dict[str, Any]): """从索引中移除记录""" # 这里可以实现索引清理逻辑 pass def _drop_table_indexes(self, table_name: str): """删除表的所有索引""" # 这里可以实现索引删除逻辑 pass # 实用方法 def list_tables(self) -> List[str]: """列出所有表""" metadata = self._get_metadata() return metadata.get("tables", []) def table_info(self, table_name: str) -> Dict[str, Any]: """获取表信息""" if not self._table_exists(table_name): raise ValueError(f"Table '{table_name}' does not exist") return self._get_table_metadata(table_name) def database_info(self) -> Dict[str, Any]: """获取数据库信息""" metadata = self._get_metadata() # 添加表的详细信息 tables_info = {} for table_name in metadata.get("tables", []): tables_info[table_name] = self._get_table_metadata(table_name) return { "database_key": self.key, "metadata": metadata, "tables": tables_info }