feat: 添加项目规则、环境配置示例及开发文档
This commit is contained in:
23
src/minenasai/core/__init__.py
Normal file
23
src/minenasai/core/__init__.py
Normal file
@@ -0,0 +1,23 @@
|
||||
"""核心模块
|
||||
|
||||
提供配置管理、日志系统、数据库等基础功能
|
||||
"""
|
||||
|
||||
from minenasai.core.config import Settings, get_settings, load_config, reset_settings
|
||||
from minenasai.core.logging import (
|
||||
AuditLogger,
|
||||
get_audit_logger,
|
||||
get_logger,
|
||||
setup_logging,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"Settings",
|
||||
"get_settings",
|
||||
"load_config",
|
||||
"reset_settings",
|
||||
"setup_logging",
|
||||
"get_logger",
|
||||
"AuditLogger",
|
||||
"get_audit_logger",
|
||||
]
|
||||
309
src/minenasai/core/config.py
Normal file
309
src/minenasai/core/config.py
Normal file
@@ -0,0 +1,309 @@
|
||||
"""配置管理模块
|
||||
|
||||
加载和验证配置文件,支持环境变量覆盖
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import json5
|
||||
from pydantic import BaseModel, Field
|
||||
from pydantic_settings import BaseSettings, SettingsConfigDict
|
||||
|
||||
|
||||
class AppConfig(BaseModel):
|
||||
"""应用基础配置"""
|
||||
|
||||
name: str = "MineNASAI"
|
||||
version: str = "0.1.0"
|
||||
debug: bool = False
|
||||
timezone: str = "Asia/Shanghai"
|
||||
|
||||
|
||||
class AgentToolsConfig(BaseModel):
|
||||
"""Agent 工具配置"""
|
||||
|
||||
allow: list[str] = Field(default_factory=list)
|
||||
deny: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AgentSandboxConfig(BaseModel):
|
||||
"""Agent 沙箱配置"""
|
||||
|
||||
mode: str = "workspace"
|
||||
allowed_paths: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class AgentConfig(BaseModel):
|
||||
"""单个 Agent 配置"""
|
||||
|
||||
id: str
|
||||
name: str = "Agent"
|
||||
workspace_path: str = "~/.config/minenasai/workspace"
|
||||
tools: AgentToolsConfig = Field(default_factory=AgentToolsConfig)
|
||||
sandbox: AgentSandboxConfig = Field(default_factory=AgentSandboxConfig)
|
||||
|
||||
|
||||
class AgentsConfig(BaseModel):
|
||||
"""Agent 全局配置"""
|
||||
|
||||
model_config = {"populate_by_name": True}
|
||||
|
||||
default_model: str = "claude-sonnet-4-20250514"
|
||||
max_tokens: int = 8192
|
||||
temperature: float = 0.7
|
||||
items: list[AgentConfig] = Field(default_factory=list, validation_alias="list")
|
||||
|
||||
|
||||
class GatewayConfig(BaseModel):
|
||||
"""Gateway 服务配置"""
|
||||
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 8000
|
||||
websocket_path: str = "/ws"
|
||||
cors_origins: list[str] = Field(default_factory=lambda: ["*"])
|
||||
|
||||
|
||||
class ChannelConfig(BaseModel):
|
||||
"""通讯渠道配置"""
|
||||
|
||||
enabled: bool = False
|
||||
webhook_path: str = ""
|
||||
|
||||
|
||||
class ChannelsConfig(BaseModel):
|
||||
"""通讯渠道集合配置"""
|
||||
|
||||
wework: ChannelConfig = Field(default_factory=ChannelConfig)
|
||||
feishu: ChannelConfig = Field(default_factory=ChannelConfig)
|
||||
|
||||
|
||||
class SSHConfig(BaseModel):
|
||||
"""SSH 连接配置"""
|
||||
|
||||
host: str = "localhost"
|
||||
port: int = 22
|
||||
connect_timeout: int = 10
|
||||
|
||||
|
||||
class WebTUIConfig(BaseModel):
|
||||
"""Web TUI 配置"""
|
||||
|
||||
enabled: bool = True
|
||||
host: str = "0.0.0.0"
|
||||
port: int = 8080
|
||||
session_timeout: int = 3600
|
||||
ssh: SSHConfig = Field(default_factory=SSHConfig)
|
||||
|
||||
|
||||
class RouterConfig(BaseModel):
|
||||
"""智能路由配置"""
|
||||
|
||||
mode: str = "agent"
|
||||
thresholds: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
|
||||
class StorageConfig(BaseModel):
|
||||
"""存储配置"""
|
||||
|
||||
database_path: str = "~/.config/minenasai/database.db"
|
||||
sessions_path: str = "~/.config/minenasai/sessions"
|
||||
logs_path: str = "~/.config/minenasai/logs"
|
||||
|
||||
|
||||
class AuditConfig(BaseModel):
|
||||
"""审计日志配置"""
|
||||
|
||||
enabled: bool = True
|
||||
path: str = "~/.config/minenasai/logs/audit.jsonl"
|
||||
|
||||
|
||||
class LoggingConfig(BaseModel):
|
||||
"""日志配置"""
|
||||
|
||||
level: str = "INFO"
|
||||
format: str = "json"
|
||||
console: bool = True
|
||||
file: bool = True
|
||||
audit: AuditConfig = Field(default_factory=AuditConfig)
|
||||
|
||||
|
||||
class ProxyConfig(BaseModel):
|
||||
"""代理配置"""
|
||||
|
||||
enabled: bool = False
|
||||
http: str = ""
|
||||
https: str = ""
|
||||
no_proxy: list[str] = Field(default_factory=list)
|
||||
# 自动检测代理端口
|
||||
auto_detect: bool = True
|
||||
fallback_ports: list[int] = Field(default_factory=lambda: [7890, 7891, 1080, 1087, 10808])
|
||||
|
||||
|
||||
class LLMConfig(BaseModel):
|
||||
"""LLM 多模型配置"""
|
||||
|
||||
# 默认提供商和模型
|
||||
default_provider: str = "anthropic"
|
||||
default_model: str = "claude-sonnet-4-20250514"
|
||||
|
||||
# Anthropic (Claude) - 境外,需代理
|
||||
anthropic_api_key: str = ""
|
||||
anthropic_base_url: str = ""
|
||||
|
||||
# OpenAI (GPT) - 境外,需代理
|
||||
openai_api_key: str = ""
|
||||
openai_base_url: str = ""
|
||||
|
||||
# DeepSeek - 国内
|
||||
deepseek_api_key: str = ""
|
||||
deepseek_base_url: str = ""
|
||||
|
||||
# 智谱 (GLM) - 国内
|
||||
zhipu_api_key: str = ""
|
||||
zhipu_base_url: str = ""
|
||||
|
||||
# MiniMax - 国内
|
||||
minimax_api_key: str = ""
|
||||
minimax_group_id: str = ""
|
||||
minimax_base_url: str = ""
|
||||
|
||||
# Moonshot (Kimi) - 国内
|
||||
moonshot_api_key: str = ""
|
||||
moonshot_base_url: str = ""
|
||||
|
||||
# Google Gemini - 境外,需代理
|
||||
gemini_api_key: str = ""
|
||||
gemini_base_url: str = ""
|
||||
|
||||
|
||||
class SecurityConfig(BaseModel):
|
||||
"""安全配置"""
|
||||
|
||||
danger_levels: dict[str, list[str]] = Field(default_factory=dict)
|
||||
require_confirmation: list[str] = Field(default_factory=lambda: ["high", "critical"])
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""应用设置,支持环境变量覆盖"""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_prefix="MINENASAI_",
|
||||
env_nested_delimiter="__",
|
||||
extra="ignore",
|
||||
)
|
||||
|
||||
# API Keys (保留兼容,优先使用 llm 配置)
|
||||
anthropic_api_key: str = ""
|
||||
|
||||
# 企业微信
|
||||
wework_corp_id: str = ""
|
||||
wework_agent_id: str = ""
|
||||
wework_secret: str = ""
|
||||
wework_token: str = ""
|
||||
wework_encoding_aes_key: str = ""
|
||||
|
||||
# 飞书
|
||||
feishu_app_id: str = ""
|
||||
feishu_app_secret: str = ""
|
||||
feishu_verification_token: str = ""
|
||||
feishu_encrypt_key: str = ""
|
||||
|
||||
# Redis
|
||||
redis_url: str = "redis://localhost:6379/0"
|
||||
|
||||
# Web TUI
|
||||
webtui_secret_key: str = "change-this-to-a-random-string"
|
||||
|
||||
# SSH
|
||||
ssh_username: str = ""
|
||||
ssh_key_path: str = "~/.ssh/id_rsa"
|
||||
|
||||
# 配置对象
|
||||
app: AppConfig = Field(default_factory=AppConfig)
|
||||
agents: AgentsConfig = Field(default_factory=AgentsConfig)
|
||||
gateway: GatewayConfig = Field(default_factory=GatewayConfig)
|
||||
channels: ChannelsConfig = Field(default_factory=ChannelsConfig)
|
||||
webtui: WebTUIConfig = Field(default_factory=WebTUIConfig)
|
||||
router: RouterConfig = Field(default_factory=RouterConfig)
|
||||
storage: StorageConfig = Field(default_factory=StorageConfig)
|
||||
logging: LoggingConfig = Field(default_factory=LoggingConfig)
|
||||
proxy: ProxyConfig = Field(default_factory=ProxyConfig)
|
||||
security: SecurityConfig = Field(default_factory=SecurityConfig)
|
||||
llm: LLMConfig = Field(default_factory=LLMConfig)
|
||||
|
||||
|
||||
def expand_path(path: str) -> Path:
|
||||
"""展开路径中的 ~ 和环境变量"""
|
||||
return Path(os.path.expanduser(os.path.expandvars(path)))
|
||||
|
||||
|
||||
def load_config(config_path: str | Path | None = None) -> Settings:
|
||||
"""加载配置文件
|
||||
|
||||
Args:
|
||||
config_path: 配置文件路径,如果为 None 则使用默认路径
|
||||
|
||||
Returns:
|
||||
Settings 配置对象
|
||||
"""
|
||||
# 默认配置路径
|
||||
if config_path is None:
|
||||
config_path = expand_path("~/.config/minenasai/config.json5")
|
||||
|
||||
config_path = Path(config_path)
|
||||
|
||||
# 如果配置文件存在,加载它
|
||||
config_data: dict[str, Any] = {}
|
||||
if config_path.exists():
|
||||
with open(config_path, encoding="utf-8") as f:
|
||||
config_data = json5.load(f)
|
||||
|
||||
# 创建 Settings 对象(会自动加载环境变量)
|
||||
settings = Settings(**config_data)
|
||||
|
||||
return settings
|
||||
|
||||
|
||||
def save_config(settings: Settings, config_path: str | Path | None = None) -> None:
|
||||
"""保存配置到文件
|
||||
|
||||
Args:
|
||||
settings: 配置对象
|
||||
config_path: 保存路径
|
||||
"""
|
||||
if config_path is None:
|
||||
config_path = expand_path("~/.config/minenasai/config.json5")
|
||||
|
||||
config_path = Path(config_path)
|
||||
config_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# 转换为 dict,排除敏感信息
|
||||
data = settings.model_dump(
|
||||
exclude={"anthropic_api_key", "wework_secret", "feishu_app_secret", "webtui_secret_key"}
|
||||
)
|
||||
|
||||
with open(config_path, "w", encoding="utf-8") as f:
|
||||
json.dump(data, f, indent=2, ensure_ascii=False)
|
||||
|
||||
|
||||
# 全局配置实例
|
||||
_settings: Settings | None = None
|
||||
|
||||
|
||||
def get_settings() -> Settings:
|
||||
"""获取全局配置实例"""
|
||||
global _settings
|
||||
if _settings is None:
|
||||
_settings = load_config()
|
||||
return _settings
|
||||
|
||||
|
||||
def reset_settings() -> None:
|
||||
"""重置全局配置(用于测试)"""
|
||||
global _settings
|
||||
_settings = None
|
||||
390
src/minenasai/core/database.py
Normal file
390
src/minenasai/core/database.py
Normal file
@@ -0,0 +1,390 @@
|
||||
"""数据库管理模块
|
||||
|
||||
使用 SQLite + aiosqlite 提供异步数据库操作
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import aiosqlite
|
||||
|
||||
from minenasai.core.config import expand_path
|
||||
|
||||
# SQL Schema 定义
|
||||
SCHEMA = """
|
||||
-- Agents表
|
||||
CREATE TABLE IF NOT EXISTS agents (
|
||||
id TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
workspace_path TEXT NOT NULL,
|
||||
model TEXT DEFAULT 'claude-sonnet-4-20250514',
|
||||
sandbox_mode TEXT DEFAULT 'workspace',
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- Sessions表
|
||||
CREATE TABLE IF NOT EXISTS sessions (
|
||||
id TEXT PRIMARY KEY,
|
||||
agent_id TEXT REFERENCES agents(id),
|
||||
channel TEXT NOT NULL,
|
||||
peer_id TEXT,
|
||||
session_key TEXT,
|
||||
status TEXT DEFAULT 'active',
|
||||
created_at INTEGER NOT NULL,
|
||||
updated_at INTEGER NOT NULL,
|
||||
metadata TEXT
|
||||
);
|
||||
|
||||
-- Messages表
|
||||
CREATE TABLE IF NOT EXISTS messages (
|
||||
id TEXT PRIMARY KEY,
|
||||
session_id TEXT REFERENCES sessions(id),
|
||||
role TEXT NOT NULL,
|
||||
content TEXT,
|
||||
tool_calls TEXT,
|
||||
timestamp INTEGER NOT NULL,
|
||||
tokens_used INTEGER DEFAULT 0
|
||||
);
|
||||
|
||||
-- Cron任务表
|
||||
CREATE TABLE IF NOT EXISTS cron_jobs (
|
||||
id TEXT PRIMARY KEY,
|
||||
agent_id TEXT REFERENCES agents(id),
|
||||
name TEXT NOT NULL,
|
||||
schedule TEXT NOT NULL,
|
||||
task TEXT NOT NULL,
|
||||
enabled INTEGER DEFAULT 1,
|
||||
last_run INTEGER,
|
||||
next_run INTEGER,
|
||||
created_at INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 审计日志表
|
||||
CREATE TABLE IF NOT EXISTS audit_logs (
|
||||
id TEXT PRIMARY KEY,
|
||||
agent_id TEXT,
|
||||
tool_name TEXT NOT NULL,
|
||||
danger_level TEXT,
|
||||
params TEXT,
|
||||
result TEXT,
|
||||
duration_ms INTEGER,
|
||||
timestamp INTEGER NOT NULL
|
||||
);
|
||||
|
||||
-- 索引
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_agent ON sessions(agent_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_sessions_status ON sessions(status);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_session ON messages(session_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_agent ON audit_logs(agent_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_logs(timestamp);
|
||||
"""
|
||||
|
||||
|
||||
class Database:
|
||||
"""异步数据库管理器"""
|
||||
|
||||
def __init__(self, db_path: str | Path | None = None) -> None:
|
||||
"""初始化数据库
|
||||
|
||||
Args:
|
||||
db_path: 数据库文件路径
|
||||
"""
|
||||
if db_path is None:
|
||||
db_path = expand_path("~/.config/minenasai/database.db")
|
||||
self.db_path = Path(db_path)
|
||||
self.db_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._conn: aiosqlite.Connection | None = None
|
||||
|
||||
async def connect(self) -> None:
|
||||
"""连接数据库"""
|
||||
self._conn = await aiosqlite.connect(self.db_path)
|
||||
self._conn.row_factory = aiosqlite.Row
|
||||
await self._conn.executescript(SCHEMA)
|
||||
await self._conn.commit()
|
||||
|
||||
async def close(self) -> None:
|
||||
"""关闭数据库连接"""
|
||||
if self._conn:
|
||||
await self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
@property
|
||||
def conn(self) -> aiosqlite.Connection:
|
||||
"""获取数据库连接"""
|
||||
if self._conn is None:
|
||||
raise RuntimeError("数据库未连接,请先调用 connect()")
|
||||
return self._conn
|
||||
|
||||
# ===== Agent 操作 =====
|
||||
|
||||
async def create_agent(
|
||||
self,
|
||||
agent_id: str,
|
||||
name: str,
|
||||
workspace_path: str,
|
||||
model: str = "claude-sonnet-4-20250514",
|
||||
sandbox_mode: str = "workspace",
|
||||
) -> dict[str, Any]:
|
||||
"""创建 Agent"""
|
||||
now = int(time.time())
|
||||
await self.conn.execute(
|
||||
"""
|
||||
INSERT INTO agents (id, name, workspace_path, model, sandbox_mode, created_at, updated_at)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(agent_id, name, workspace_path, model, sandbox_mode, now, now),
|
||||
)
|
||||
await self.conn.commit()
|
||||
return {
|
||||
"id": agent_id,
|
||||
"name": name,
|
||||
"workspace_path": workspace_path,
|
||||
"model": model,
|
||||
"sandbox_mode": sandbox_mode,
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
}
|
||||
|
||||
async def get_agent(self, agent_id: str) -> dict[str, Any] | None:
|
||||
"""获取 Agent"""
|
||||
cursor = await self.conn.execute(
|
||||
"SELECT * FROM agents WHERE id = ?", (agent_id,)
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
async def list_agents(self) -> list[dict[str, Any]]:
|
||||
"""列出所有 Agent"""
|
||||
cursor = await self.conn.execute("SELECT * FROM agents ORDER BY created_at DESC")
|
||||
rows = await cursor.fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
# ===== Session 操作 =====
|
||||
|
||||
async def create_session(
|
||||
self,
|
||||
agent_id: str,
|
||||
channel: str,
|
||||
peer_id: str | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""创建会话"""
|
||||
session_id = str(uuid.uuid4())
|
||||
session_key = f"{agent_id}:{channel}:{session_id[:8]}"
|
||||
now = int(time.time())
|
||||
|
||||
await self.conn.execute(
|
||||
"""
|
||||
INSERT INTO sessions (id, agent_id, channel, peer_id, session_key, status, created_at, updated_at, metadata)
|
||||
VALUES (?, ?, ?, ?, ?, 'active', ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
session_id,
|
||||
agent_id,
|
||||
channel,
|
||||
peer_id,
|
||||
session_key,
|
||||
now,
|
||||
now,
|
||||
json.dumps(metadata) if metadata else None,
|
||||
),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
return {
|
||||
"id": session_id,
|
||||
"agent_id": agent_id,
|
||||
"channel": channel,
|
||||
"peer_id": peer_id,
|
||||
"session_key": session_key,
|
||||
"status": "active",
|
||||
"created_at": now,
|
||||
"updated_at": now,
|
||||
"metadata": metadata,
|
||||
}
|
||||
|
||||
async def get_session(self, session_id: str) -> dict[str, Any] | None:
|
||||
"""获取会话"""
|
||||
cursor = await self.conn.execute(
|
||||
"SELECT * FROM sessions WHERE id = ?", (session_id,)
|
||||
)
|
||||
row = await cursor.fetchone()
|
||||
if row:
|
||||
data = dict(row)
|
||||
if data.get("metadata"):
|
||||
data["metadata"] = json.loads(data["metadata"])
|
||||
return data
|
||||
return None
|
||||
|
||||
async def update_session_status(self, session_id: str, status: str) -> None:
|
||||
"""更新会话状态"""
|
||||
now = int(time.time())
|
||||
await self.conn.execute(
|
||||
"UPDATE sessions SET status = ?, updated_at = ? WHERE id = ?",
|
||||
(status, now, session_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
async def list_active_sessions(self, agent_id: str | None = None) -> list[dict[str, Any]]:
|
||||
"""列出活跃会话"""
|
||||
if agent_id:
|
||||
cursor = await self.conn.execute(
|
||||
"SELECT * FROM sessions WHERE status = 'active' AND agent_id = ? ORDER BY updated_at DESC",
|
||||
(agent_id,),
|
||||
)
|
||||
else:
|
||||
cursor = await self.conn.execute(
|
||||
"SELECT * FROM sessions WHERE status = 'active' ORDER BY updated_at DESC"
|
||||
)
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for row in rows:
|
||||
data = dict(row)
|
||||
if data.get("metadata"):
|
||||
data["metadata"] = json.loads(data["metadata"])
|
||||
results.append(data)
|
||||
return results
|
||||
|
||||
# ===== Message 操作 =====
|
||||
|
||||
async def add_message(
|
||||
self,
|
||||
session_id: str,
|
||||
role: str,
|
||||
content: str | None = None,
|
||||
tool_calls: list[dict[str, Any]] | None = None,
|
||||
tokens_used: int = 0,
|
||||
) -> dict[str, Any]:
|
||||
"""添加消息"""
|
||||
message_id = str(uuid.uuid4())
|
||||
now = int(time.time())
|
||||
|
||||
await self.conn.execute(
|
||||
"""
|
||||
INSERT INTO messages (id, session_id, role, content, tool_calls, timestamp, tokens_used)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
message_id,
|
||||
session_id,
|
||||
role,
|
||||
content,
|
||||
json.dumps(tool_calls) if tool_calls else None,
|
||||
now,
|
||||
tokens_used,
|
||||
),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
# 更新会话的 updated_at
|
||||
await self.conn.execute(
|
||||
"UPDATE sessions SET updated_at = ? WHERE id = ?",
|
||||
(now, session_id),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
return {
|
||||
"id": message_id,
|
||||
"session_id": session_id,
|
||||
"role": role,
|
||||
"content": content,
|
||||
"tool_calls": tool_calls,
|
||||
"timestamp": now,
|
||||
"tokens_used": tokens_used,
|
||||
}
|
||||
|
||||
async def get_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: int = 100,
|
||||
before_timestamp: int | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""获取会话消息"""
|
||||
if before_timestamp:
|
||||
cursor = await self.conn.execute(
|
||||
"""
|
||||
SELECT * FROM messages
|
||||
WHERE session_id = ? AND timestamp < ?
|
||||
ORDER BY timestamp DESC LIMIT ?
|
||||
""",
|
||||
(session_id, before_timestamp, limit),
|
||||
)
|
||||
else:
|
||||
cursor = await self.conn.execute(
|
||||
"""
|
||||
SELECT * FROM messages
|
||||
WHERE session_id = ?
|
||||
ORDER BY timestamp DESC LIMIT ?
|
||||
""",
|
||||
(session_id, limit),
|
||||
)
|
||||
|
||||
rows = await cursor.fetchall()
|
||||
results = []
|
||||
for row in reversed(rows): # 按时间正序返回
|
||||
data = dict(row)
|
||||
if data.get("tool_calls"):
|
||||
data["tool_calls"] = json.loads(data["tool_calls"])
|
||||
results.append(data)
|
||||
return results
|
||||
|
||||
# ===== 审计日志 =====
|
||||
|
||||
async def add_audit_log(
|
||||
self,
|
||||
agent_id: str | None,
|
||||
tool_name: str,
|
||||
danger_level: str,
|
||||
params: dict[str, Any] | None = None,
|
||||
result: str | None = None,
|
||||
duration_ms: int | None = None,
|
||||
) -> None:
|
||||
"""添加审计日志"""
|
||||
log_id = str(uuid.uuid4())
|
||||
now = int(time.time())
|
||||
|
||||
await self.conn.execute(
|
||||
"""
|
||||
INSERT INTO audit_logs (id, agent_id, tool_name, danger_level, params, result, duration_ms, timestamp)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
||||
""",
|
||||
(
|
||||
log_id,
|
||||
agent_id,
|
||||
tool_name,
|
||||
danger_level,
|
||||
json.dumps(params) if params else None,
|
||||
result,
|
||||
duration_ms,
|
||||
now,
|
||||
),
|
||||
)
|
||||
await self.conn.commit()
|
||||
|
||||
|
||||
# 全局数据库实例
|
||||
_db: Database | None = None
|
||||
|
||||
|
||||
async def get_database() -> Database:
|
||||
"""获取全局数据库实例"""
|
||||
global _db
|
||||
if _db is None:
|
||||
_db = Database()
|
||||
await _db.connect()
|
||||
return _db
|
||||
|
||||
|
||||
async def close_database() -> None:
|
||||
"""关闭全局数据库连接"""
|
||||
global _db
|
||||
if _db:
|
||||
await _db.close()
|
||||
_db = None
|
||||
212
src/minenasai/core/logging.py
Normal file
212
src/minenasai/core/logging.py
Normal file
@@ -0,0 +1,212 @@
|
||||
"""日志系统模块
|
||||
|
||||
使用 structlog 提供结构化 JSON 日志
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import structlog
|
||||
from structlog.types import Processor
|
||||
|
||||
from minenasai.core.config import LoggingConfig, expand_path
|
||||
|
||||
|
||||
def setup_logging(config: LoggingConfig | None = None) -> None:
|
||||
"""初始化日志系统
|
||||
|
||||
Args:
|
||||
config: 日志配置,如果为 None 则使用默认配置
|
||||
"""
|
||||
if config is None:
|
||||
config = LoggingConfig()
|
||||
|
||||
# 设置日志级别
|
||||
log_level = getattr(logging, config.level.upper(), logging.INFO)
|
||||
|
||||
# 配置处理器链
|
||||
shared_processors: list[Processor] = [
|
||||
structlog.contextvars.merge_contextvars,
|
||||
structlog.processors.add_log_level,
|
||||
structlog.processors.TimeStamper(fmt="iso"),
|
||||
structlog.processors.StackInfoRenderer(),
|
||||
structlog.processors.UnicodeDecoder(),
|
||||
]
|
||||
|
||||
if config.format == "json":
|
||||
# JSON 格式输出
|
||||
renderer: Processor = structlog.processors.JSONRenderer(ensure_ascii=False)
|
||||
else:
|
||||
# 控制台友好格式
|
||||
renderer = structlog.dev.ConsoleRenderer(colors=True)
|
||||
|
||||
# 配置 structlog
|
||||
structlog.configure(
|
||||
processors=[
|
||||
*shared_processors,
|
||||
structlog.processors.format_exc_info,
|
||||
renderer,
|
||||
],
|
||||
wrapper_class=structlog.make_filtering_bound_logger(log_level),
|
||||
context_class=dict,
|
||||
logger_factory=structlog.PrintLoggerFactory(),
|
||||
cache_logger_on_first_use=True,
|
||||
)
|
||||
|
||||
# 配置标准库 logging
|
||||
logging.basicConfig(
|
||||
format="%(message)s",
|
||||
stream=sys.stdout,
|
||||
level=log_level,
|
||||
)
|
||||
|
||||
# 配置文件日志
|
||||
if config.file:
|
||||
logs_path = expand_path(config.audit.path).parent
|
||||
logs_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
file_handler = logging.FileHandler(
|
||||
logs_path / "app.log",
|
||||
encoding="utf-8",
|
||||
)
|
||||
file_handler.setLevel(log_level)
|
||||
logging.getLogger().addHandler(file_handler)
|
||||
|
||||
|
||||
def get_logger(name: str | None = None) -> structlog.BoundLogger:
|
||||
"""获取日志记录器
|
||||
|
||||
Args:
|
||||
name: 日志记录器名称
|
||||
|
||||
Returns:
|
||||
structlog 绑定的日志记录器
|
||||
"""
|
||||
return structlog.get_logger(name)
|
||||
|
||||
|
||||
class AuditLogger:
|
||||
"""审计日志记录器
|
||||
|
||||
记录工具调用、权限操作等需要审计的事件
|
||||
"""
|
||||
|
||||
def __init__(self, audit_path: str | Path | None = None) -> None:
|
||||
"""初始化审计日志
|
||||
|
||||
Args:
|
||||
audit_path: 审计日志文件路径
|
||||
"""
|
||||
if audit_path is None:
|
||||
audit_path = expand_path("~/.config/minenasai/logs/audit.jsonl")
|
||||
self.audit_path = Path(audit_path)
|
||||
self.audit_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
self._logger = get_logger("audit")
|
||||
|
||||
def log_tool_call(
|
||||
self,
|
||||
agent_id: str,
|
||||
tool_name: str,
|
||||
params: dict[str, Any],
|
||||
danger_level: str,
|
||||
result: str | None = None,
|
||||
duration_ms: int | None = None,
|
||||
error: str | None = None,
|
||||
) -> None:
|
||||
"""记录工具调用
|
||||
|
||||
Args:
|
||||
agent_id: Agent ID
|
||||
tool_name: 工具名称
|
||||
params: 调用参数
|
||||
danger_level: 危险等级
|
||||
result: 执行结果
|
||||
duration_ms: 执行耗时(毫秒)
|
||||
error: 错误信息
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
|
||||
record = {
|
||||
"timestamp": time.time(),
|
||||
"type": "tool_call",
|
||||
"agent_id": agent_id,
|
||||
"tool_name": tool_name,
|
||||
"params": params,
|
||||
"danger_level": danger_level,
|
||||
"result": result,
|
||||
"duration_ms": duration_ms,
|
||||
"error": error,
|
||||
}
|
||||
|
||||
# 写入 JSONL 文件
|
||||
with open(self.audit_path, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
# 同时记录到结构化日志
|
||||
self._logger.info(
|
||||
"tool_call",
|
||||
agent_id=agent_id,
|
||||
tool_name=tool_name,
|
||||
danger_level=danger_level,
|
||||
duration_ms=duration_ms,
|
||||
error=error,
|
||||
)
|
||||
|
||||
def log_auth_event(
|
||||
self,
|
||||
event_type: str,
|
||||
user_id: str | None = None,
|
||||
channel: str | None = None,
|
||||
success: bool = True,
|
||||
details: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
"""记录认证事件
|
||||
|
||||
Args:
|
||||
event_type: 事件类型(login, logout, token_refresh 等)
|
||||
user_id: 用户 ID
|
||||
channel: 渠道(wework, feishu, webtui)
|
||||
success: 是否成功
|
||||
details: 额外详情
|
||||
"""
|
||||
import json
|
||||
import time
|
||||
|
||||
record = {
|
||||
"timestamp": time.time(),
|
||||
"type": "auth_event",
|
||||
"event_type": event_type,
|
||||
"user_id": user_id,
|
||||
"channel": channel,
|
||||
"success": success,
|
||||
"details": details or {},
|
||||
}
|
||||
|
||||
with open(self.audit_path, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
log_method = self._logger.info if success else self._logger.warning
|
||||
log_method(
|
||||
"auth_event",
|
||||
event_type=event_type,
|
||||
user_id=user_id,
|
||||
channel=channel,
|
||||
success=success,
|
||||
)
|
||||
|
||||
|
||||
# 全局审计日志实例
|
||||
_audit_logger: AuditLogger | None = None
|
||||
|
||||
|
||||
def get_audit_logger() -> AuditLogger:
|
||||
"""获取全局审计日志实例"""
|
||||
global _audit_logger
|
||||
if _audit_logger is None:
|
||||
_audit_logger = AuditLogger()
|
||||
return _audit_logger
|
||||
282
src/minenasai/core/session_store.py
Normal file
282
src/minenasai/core/session_store.py
Normal file
@@ -0,0 +1,282 @@
|
||||
"""会话存储模块
|
||||
|
||||
使用 JSONL 格式存储完整的对话历史
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import time
|
||||
from pathlib import Path
|
||||
from typing import Any, Iterator
|
||||
|
||||
from minenasai.core.config import expand_path
|
||||
|
||||
|
||||
class SessionStore:
|
||||
"""JSONL 会话存储
|
||||
|
||||
每个会话一个 .jsonl 文件,存储完整的消息历史
|
||||
"""
|
||||
|
||||
def __init__(self, sessions_path: str | Path | None = None) -> None:
|
||||
"""初始化会话存储
|
||||
|
||||
Args:
|
||||
sessions_path: 会话存储目录
|
||||
"""
|
||||
if sessions_path is None:
|
||||
sessions_path = expand_path("~/.config/minenasai/sessions")
|
||||
self.sessions_path = Path(sessions_path)
|
||||
self.sessions_path.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def _get_session_file(self, session_id: str) -> Path:
|
||||
"""获取会话文件路径"""
|
||||
return self.sessions_path / f"{session_id}.jsonl"
|
||||
|
||||
def append_message(
|
||||
self,
|
||||
session_id: str,
|
||||
role: str,
|
||||
content: str | None = None,
|
||||
tool_calls: list[dict[str, Any]] | None = None,
|
||||
tool_results: list[dict[str, Any]] | None = None,
|
||||
metadata: dict[str, Any] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""追加消息到会话文件
|
||||
|
||||
Args:
|
||||
session_id: 会话 ID
|
||||
role: 角色 (user, assistant, system, tool)
|
||||
content: 消息内容
|
||||
tool_calls: 工具调用列表
|
||||
tool_results: 工具结果列表
|
||||
metadata: 额外元数据
|
||||
|
||||
Returns:
|
||||
保存的消息记录
|
||||
"""
|
||||
record = {
|
||||
"timestamp": time.time(),
|
||||
"role": role,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
if tool_calls:
|
||||
record["tool_calls"] = tool_calls
|
||||
if tool_results:
|
||||
record["tool_results"] = tool_results
|
||||
if metadata:
|
||||
record["metadata"] = metadata
|
||||
|
||||
session_file = self._get_session_file(session_id)
|
||||
with open(session_file, "a", encoding="utf-8") as f:
|
||||
f.write(json.dumps(record, ensure_ascii=False) + "\n")
|
||||
|
||||
return record
|
||||
|
||||
def read_messages(
|
||||
self,
|
||||
session_id: str,
|
||||
limit: int | None = None,
|
||||
since_timestamp: float | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""读取会话消息
|
||||
|
||||
Args:
|
||||
session_id: 会话 ID
|
||||
limit: 最大返回数量(从最新开始)
|
||||
since_timestamp: 只返回此时间戳之后的消息
|
||||
|
||||
Returns:
|
||||
消息列表
|
||||
"""
|
||||
session_file = self._get_session_file(session_id)
|
||||
if not session_file.exists():
|
||||
return []
|
||||
|
||||
messages = []
|
||||
with open(session_file, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
if since_timestamp and record.get("timestamp", 0) <= since_timestamp:
|
||||
continue
|
||||
messages.append(record)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
if limit:
|
||||
messages = messages[-limit:]
|
||||
|
||||
return messages
|
||||
|
||||
def iter_messages(self, session_id: str) -> Iterator[dict[str, Any]]:
|
||||
"""迭代读取会话消息(节省内存)
|
||||
|
||||
Args:
|
||||
session_id: 会话 ID
|
||||
|
||||
Yields:
|
||||
消息记录
|
||||
"""
|
||||
session_file = self._get_session_file(session_id)
|
||||
if not session_file.exists():
|
||||
return
|
||||
|
||||
with open(session_file, encoding="utf-8") as f:
|
||||
for line in f:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
yield json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
def get_conversation_for_api(
|
||||
self,
|
||||
session_id: str,
|
||||
max_messages: int = 50,
|
||||
max_tokens_estimate: int = 100000,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""获取用于 API 调用的对话历史
|
||||
|
||||
转换为 Anthropic API 格式,并进行截断
|
||||
|
||||
Args:
|
||||
session_id: 会话 ID
|
||||
max_messages: 最大消息数
|
||||
max_tokens_estimate: 预估最大 token 数
|
||||
|
||||
Returns:
|
||||
API 格式的消息列表
|
||||
"""
|
||||
messages = self.read_messages(session_id, limit=max_messages)
|
||||
|
||||
# 转换为 API 格式
|
||||
api_messages = []
|
||||
estimated_tokens = 0
|
||||
|
||||
for msg in messages:
|
||||
role = msg.get("role")
|
||||
content = msg.get("content")
|
||||
|
||||
# 跳过 system 消息(需要单独处理)
|
||||
if role == "system":
|
||||
continue
|
||||
|
||||
# 简单估算 token 数(约 4 字符/token)
|
||||
if content:
|
||||
estimated_tokens += len(content) // 4
|
||||
|
||||
if estimated_tokens > max_tokens_estimate:
|
||||
break
|
||||
|
||||
api_msg: dict[str, Any] = {"role": role}
|
||||
|
||||
if content:
|
||||
api_msg["content"] = content
|
||||
|
||||
# 处理工具调用
|
||||
if msg.get("tool_calls"):
|
||||
api_msg["content"] = [
|
||||
{"type": "text", "text": content or ""}
|
||||
] if content else []
|
||||
for tc in msg["tool_calls"]:
|
||||
api_msg["content"].append({
|
||||
"type": "tool_use",
|
||||
"id": tc.get("id"),
|
||||
"name": tc.get("name"),
|
||||
"input": tc.get("input", {}),
|
||||
})
|
||||
|
||||
# 处理工具结果
|
||||
if msg.get("tool_results"):
|
||||
api_msg["content"] = []
|
||||
for tr in msg["tool_results"]:
|
||||
api_msg["content"].append({
|
||||
"type": "tool_result",
|
||||
"tool_use_id": tr.get("tool_use_id"),
|
||||
"content": tr.get("content", ""),
|
||||
})
|
||||
|
||||
api_messages.append(api_msg)
|
||||
|
||||
return api_messages
|
||||
|
||||
def delete_session(self, session_id: str) -> bool:
|
||||
"""删除会话文件
|
||||
|
||||
Args:
|
||||
session_id: 会话 ID
|
||||
|
||||
Returns:
|
||||
是否成功删除
|
||||
"""
|
||||
session_file = self._get_session_file(session_id)
|
||||
if session_file.exists():
|
||||
session_file.unlink()
|
||||
return True
|
||||
return False
|
||||
|
||||
def list_sessions(self) -> list[str]:
|
||||
"""列出所有会话 ID
|
||||
|
||||
Returns:
|
||||
会话 ID 列表
|
||||
"""
|
||||
return [f.stem for f in self.sessions_path.glob("*.jsonl")]
|
||||
|
||||
def get_session_stats(self, session_id: str) -> dict[str, Any]:
|
||||
"""获取会话统计信息
|
||||
|
||||
Args:
|
||||
session_id: 会话 ID
|
||||
|
||||
Returns:
|
||||
统计信息
|
||||
"""
|
||||
session_file = self._get_session_file(session_id)
|
||||
if not session_file.exists():
|
||||
return {"exists": False}
|
||||
|
||||
message_count = 0
|
||||
first_timestamp = None
|
||||
last_timestamp = None
|
||||
role_counts: dict[str, int] = {}
|
||||
|
||||
for msg in self.iter_messages(session_id):
|
||||
message_count += 1
|
||||
ts = msg.get("timestamp")
|
||||
role = msg.get("role", "unknown")
|
||||
|
||||
if first_timestamp is None:
|
||||
first_timestamp = ts
|
||||
last_timestamp = ts
|
||||
|
||||
role_counts[role] = role_counts.get(role, 0) + 1
|
||||
|
||||
return {
|
||||
"exists": True,
|
||||
"file_size": session_file.stat().st_size,
|
||||
"message_count": message_count,
|
||||
"first_timestamp": first_timestamp,
|
||||
"last_timestamp": last_timestamp,
|
||||
"role_counts": role_counts,
|
||||
}
|
||||
|
||||
|
||||
# 全局会话存储实例
|
||||
_session_store: SessionStore | None = None
|
||||
|
||||
|
||||
def get_session_store() -> SessionStore:
|
||||
"""获取全局会话存储实例"""
|
||||
global _session_store
|
||||
if _session_store is None:
|
||||
_session_store = SessionStore()
|
||||
return _session_store
|
||||
Reference in New Issue
Block a user