636 lines
21 KiB
Python
636 lines
21 KiB
Python
"""
|
||
Neural Sync - Enhanced Code Auditor Service
|
||
|
||
增强版代码审计服务 - 支持严格模式的自动合并
|
||
"""
|
||
|
||
import asyncio
|
||
import re
|
||
import logging
|
||
from typing import Dict, List, Optional, Any, Tuple
|
||
from pydantic import BaseModel, Field
|
||
from datetime import datetime
|
||
from enum import Enum
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
|
||
class AuditSeverity(str, Enum):
|
||
"""审计问题严重程度"""
|
||
CRITICAL = "critical" # 阻止合并
|
||
HIGH = "high" # 大幅扣分
|
||
MEDIUM = "medium" # 中度扣分
|
||
LOW = "low" # 轻微扣分
|
||
INFO = "info" # 仅提示
|
||
|
||
|
||
class AuditIssue(BaseModel):
|
||
"""审计问题"""
|
||
severity: AuditSeverity
|
||
category: str # security, performance, style, logic, etc.
|
||
file_path: str
|
||
line_number: Optional[int] = None
|
||
message: str
|
||
suggestion: Optional[str] = None
|
||
score_penalty: float = 0.0
|
||
|
||
|
||
class AuditResult(BaseModel):
|
||
"""审计结果"""
|
||
signal_id: Optional[str] = None
|
||
commit_sha: Optional[str] = None
|
||
task_name: Optional[str] = None
|
||
passed: bool
|
||
score: float = Field(default=0.0, ge=0.0, le=100.0)
|
||
issues: List[AuditIssue] = Field(default_factory=list)
|
||
summary: str = ""
|
||
details: Dict[str, Any] = Field(default_factory=dict)
|
||
timestamp: float = Field(default_factory=lambda: datetime.utcnow().timestamp())
|
||
|
||
|
||
class SecurityIssue(BaseModel):
|
||
"""安全问题"""
|
||
type: str # sql_injection, xss, csrf, etc.
|
||
severity: AuditSeverity
|
||
file_path: str
|
||
line_number: int
|
||
code_snippet: str
|
||
description: str
|
||
fix_suggestion: str
|
||
|
||
|
||
class CodeAuditor:
|
||
"""
|
||
增强版代码审计器
|
||
|
||
特性:
|
||
- 安全漏洞扫描
|
||
- 代码质量评估
|
||
- 性能问题检测
|
||
- 最佳实践检查
|
||
- 严格评分机制（98+ 分阈值）
|
||
"""
|
||
|
||
# 审计权重
|
||
WEIGHTS = {
|
||
"security": 30, # 安全性最重要
|
||
"functionality": 25, # 功能正确性
|
||
"performance": 15, # 性能优化
|
||
"style": 10, # 代码风格
|
||
"documentation": 10, # 文档完整性
|
||
"testing": 10, # 测试覆盖
|
||
}
|
||
|
||
# 安全模式库
|
||
SECURITY_PATTERNS = {
|
||
"sql_injection": [
|
||
r'execute\s*\(\s*["\'].*?\+.*?["\']', # 字符串拼接 SQL
|
||
r'cursor\.execute\s*\(\s*["\'].*?%s.*?["\']', # %s 格式化
|
||
r'\$\{.*\}.*?(?:SELECT|INSERT|UPDATE|DELETE)', # 模板字符串 SQL
|
||
],
|
||
"xss": [
|
||
r'innerHTML\s*=\s*.*?\+.*?(?:request|input|form)',
|
||
r'eval\s*\(',
|
||
r'document\.write\s*\(',
|
||
],
|
||
"hardcoded_secrets": [
|
||
r'(?:password|secret|api_key|token)\s*=\s*["\'][^"\']+["\']',
|
||
r'Bearer\s+[A-Za-z0-9]{20,}',
|
||
],
|
||
"command_injection": [
|
||
r'subprocess\.call\s*\(\s*["\'].*?\+.*?["\']',
|
||
r'os\.system\s*\(\s*["\'].*?\+.*?["\']',
|
||
],
|
||
"path_traversal": [
|
||
r'open\s*\(\s*["\'].*?\.\.\/.*?["\']',
|
||
r'Path\s*\(\s*["\'].*?\.\.\/.*?["\']',
|
||
],
|
||
}
|
||
|
||
# 性能问题模式
|
||
PERFORMANCE_PATTERNS = {
|
||
"n_plus_1": [
|
||
r'for\s+\w+\s+in\s+range\s*\(\s*len\s*\(',
|
||
r'\[[^\]]+\]\s*\.append\s*\(.*?\)\s*for\s+\w+\s+in\s+',
|
||
],
|
||
"inefficient_string_concat": [
|
||
r'["\'][^"\']+["\']\s*\+.*?\+',
|
||
],
|
||
"unused_import": [],
|
||
"large_file": None, # 文件行数检查
|
||
}
|
||
|
||
# 代码风格检查
|
||
STYLE_PATTERNS = {
|
||
"line_too_long": 100, # 最大行长度
|
||
"function_too_long": 50, # 最大函数行数
|
||
"class_too_long": 200, # 最大类行数
|
||
}
|
||
|
||
def __init__(self, llm_service):
|
||
self.llm = llm_service
|
||
|
||
async def audit_code_change(
|
||
self,
|
||
task_name: str,
|
||
commit_message: str,
|
||
code_diff: str,
|
||
signal_id: Optional[str] = None,
|
||
commit_sha: Optional[str] = None,
|
||
file_contexts: Optional[Dict[str, str]] = None,
|
||
) -> AuditResult:
|
||
"""
|
||
审计代码变更
|
||
|
||
Args:
|
||
task_name: 任务名称
|
||
commit_message: 提交消息
|
||
code_diff: 代码 diff
|
||
signal_id: 关联的信号 ID
|
||
commit_sha: 提交 SHA
|
||
file_contexts: 文件上下文（完整文件内容）
|
||
|
||
Returns:
|
||
审计结果
|
||
"""
|
||
issues = []
|
||
|
||
try:
|
||
# 解析 diff 获取变更的文件
|
||
changed_files = self._parse_diff(code_diff)
|
||
|
||
# 对每个变更的文件进行审计
|
||
for file_path, file_changes in changed_files.items():
|
||
# 获取文件内容（如果提供了上下文）
|
||
full_content = file_contexts.get(file_path) if file_contexts else None
|
||
|
||
# 安全审计
|
||
security_issues = await self._audit_security(
|
||
file_path, file_changes, full_content
|
||
)
|
||
issues.extend(security_issues)
|
||
|
||
# 性能审计
|
||
perf_issues = self._audit_performance(
|
||
file_path, file_changes, full_content
|
||
)
|
||
issues.extend(perf_issues)
|
||
|
||
# 代码风格审计
|
||
style_issues = self._audit_style(
|
||
file_path, file_changes, full_content
|
||
)
|
||
issues.extend(style_issues)
|
||
|
||
# 功能正确性审计（使用 AI）
|
||
functional_issues = await self._audit_functionality(
|
||
task_name, file_path, file_changes, full_content
|
||
)
|
||
issues.extend(functional_issues)
|
||
|
||
# AI 深度审计
|
||
ai_issues = await self._ai_deep_audit(
|
||
task_name, commit_message, code_diff, changed_files
|
||
)
|
||
issues.extend(ai_issues)
|
||
|
||
# 计算总分
|
||
score = self._calculate_score(issues)
|
||
|
||
# 生成摘要
|
||
summary = self._generate_summary(issues, score)
|
||
|
||
# 判断是否通过（严格模式：98分）
|
||
passed = score >= 98.0 and not any(
|
||
i.severity == AuditSeverity.CRITICAL for i in issues
|
||
)
|
||
|
||
return AuditResult(
|
||
signal_id=signal_id,
|
||
commit_sha=commit_sha,
|
||
task_name=task_name,
|
||
passed=passed,
|
||
score=score,
|
||
issues=issues,
|
||
summary=summary,
|
||
details={
|
||
"changed_files_count": len(changed_files),
|
||
"issues_by_severity": self._count_issues_by_severity(issues),
|
||
"issues_by_category": self._count_issues_by_category(issues),
|
||
"changed_files": list(changed_files.keys()),
|
||
},
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"Audit failed: {e}", exc_info=True)
|
||
return AuditResult(
|
||
signal_id=signal_id,
|
||
commit_sha=commit_sha,
|
||
task_name=task_name,
|
||
passed=False,
|
||
score=0.0,
|
||
issues=[
|
||
AuditIssue(
|
||
severity=AuditSeverity.CRITICAL,
|
||
category="system",
|
||
file_path="unknown",
|
||
message=f"审计过程出错: {str(e)}",
|
||
score_penalty=100.0,
|
||
)
|
||
],
|
||
summary=f"审计失败: {str(e)}",
|
||
)
|
||
|
||
def _parse_diff(self, diff: str) -> Dict[str, str]:
|
||
"""解析 unified diff 格式"""
|
||
files = {}
|
||
current_file = None
|
||
current_content = []
|
||
|
||
for line in diff.split("\n"):
|
||
if line.startswith("diff --git"):
|
||
# 新文件开始
|
||
if current_file and current_content:
|
||
files[current_file] = "\n".join(current_content)
|
||
|
||
# 提取文件路径
|
||
parts = line.split()
|
||
if len(parts) >= 4:
|
||
current_file = parts[3].lstrip("b/")
|
||
else:
|
||
current_file = parts[2].lstrip("b/")
|
||
current_content = []
|
||
|
||
elif line.startswith("@@") or line.startswith("---") or line.startswith("+++"):
|
||
continue
|
||
elif current_file:
|
||
current_content.append(line)
|
||
|
||
# 保存最后一个文件
|
||
if current_file and current_content:
|
||
files[current_file] = "\n".join(current_content)
|
||
|
||
return files
|
||
|
||
async def _audit_security(
|
||
self, file_path: str, file_changes: str, full_content: Optional[str]
|
||
) -> List[AuditIssue]:
|
||
"""安全审计"""
|
||
issues = []
|
||
content = full_content or file_changes
|
||
|
||
# 检查常见安全问题
|
||
for vuln_type, patterns in self.SECURITY_PATTERNS.items():
|
||
for pattern in patterns:
|
||
matches = re.finditer(pattern, content, re.IGNORECASE | re.MULTILINE)
|
||
for match in matches:
|
||
line_num = content[:match.start()].count("\n") + 1
|
||
|
||
# 计算扣分
|
||
if vuln_type in ["sql_injection", "xss", "command_injection"]:
|
||
severity = AuditSeverity.CRITICAL
|
||
penalty = 20.0
|
||
elif vuln_type == "hardcoded_secrets":
|
||
severity = AuditSeverity.HIGH
|
||
penalty = 10.0
|
||
elif vuln_type == "path_traversal":
|
||
severity = AuditSeverity.HIGH
|
||
penalty = 15.0
|
||
else:
|
||
severity = AuditSeverity.MEDIUM
|
||
penalty = 5.0
|
||
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=severity,
|
||
category="security",
|
||
file_path=file_path,
|
||
line_number=line_num,
|
||
message=f"潜在{vuln_type}漏洞",
|
||
suggestion=f"使用参数化查询/输入验证来避免{vuln_type}",
|
||
score_penalty=penalty,
|
||
)
|
||
)
|
||
|
||
return issues
|
||
|
||
def _audit_performance(
|
||
self, file_path: str, file_changes: str, full_content: Optional[str]
|
||
) -> List[AuditIssue]:
|
||
"""性能审计"""
|
||
issues = []
|
||
content = full_content or file_changes
|
||
|
||
# 检查 N+1 问题
|
||
for pattern in self.PERFORMANCE_PATTERNS.get("n_plus_1", []):
|
||
matches = re.finditer(pattern, content, re.MULTILINE)
|
||
for match in matches:
|
||
line_num = content[:match.start()].count("\n") + 1
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=AuditSeverity.MEDIUM,
|
||
category="performance",
|
||
file_path=file_path,
|
||
line_number=line_num,
|
||
message="可能的 N+1 查询问题",
|
||
suggestion="考虑使用 bulk_create 或 select_related 优化",
|
||
score_penalty=5.0,
|
||
)
|
||
)
|
||
|
||
# 检查低效字符串拼接
|
||
for pattern in self.PERFORMANCE_PATTERNS.get("inefficient_string_concat", []):
|
||
matches = re.finditer(pattern, content)
|
||
for match in matches:
|
||
line_num = content[:match.start()].count("\n") + 1
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=AuditSeverity.LOW,
|
||
category="performance",
|
||
file_path=file_path,
|
||
line_number=line_num,
|
||
message="低效的字符串拼接",
|
||
suggestion="使用 f-string 或 join()",
|
||
score_penalty=2.0,
|
||
)
|
||
)
|
||
|
||
# 检查文件大小
|
||
if full_content:
|
||
line_count = len(full_content.split("\n"))
|
||
if line_count > 500:
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=AuditSeverity.MEDIUM,
|
||
category="performance",
|
||
file_path=file_path,
|
||
message=f"文件过大 ({line_count} 行)",
|
||
suggestion="考虑拆分为更小的模块",
|
||
score_penalty=5.0,
|
||
)
|
||
)
|
||
|
||
return issues
|
||
|
||
def _audit_style(
|
||
self, file_path: str, file_changes: str, full_content: Optional[str]
|
||
) -> List[AuditIssue]:
|
||
"""代码风格审计"""
|
||
issues = []
|
||
content = full_content or file_changes
|
||
|
||
# 检查行长度
|
||
max_length = self.STYLE_PATTERNS["line_too_long"]
|
||
for i, line in enumerate(content.split("\n"), 1):
|
||
if len(line) > max_length:
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=AuditSeverity.INFO,
|
||
category="style",
|
||
file_path=file_path,
|
||
line_number=i,
|
||
message=f"行过长 ({len(line)} > {max_length})",
|
||
suggestion="拆分为多行",
|
||
score_penalty=1.0,
|
||
)
|
||
)
|
||
|
||
return issues
|
||
|
||
async def _audit_functionality(
|
||
self, task_name: str, file_path: str, file_changes: str, full_content: Optional[str]
|
||
) -> List[AuditIssue]:
|
||
"""功能正确性审计（使用 AI）"""
|
||
issues = []
|
||
|
||
# 构建 AI 审计 prompt
|
||
prompt = f"""作为一位资深代码审查员，请审计以下代码变更：
|
||
|
||
任务需求：{task_name}
|
||
|
||
文件：{file_path}
|
||
|
||
代码变更：
|
||
```diff
|
||
{file_changes}
|
||
```
|
||
|
||
{f'完整文件内容：\n```{self._detect_language(file_path)}\n{full_content}\n```' if full_content else ''}
|
||
|
||
请从以下维度进行审查：
|
||
1. 功能正确性：代码是否正确实现了需求
|
||
2. 逻辑完整性：是否考虑了边界情况
|
||
3. 错误处理：是否有适当的异常处理
|
||
4. 资源管理：是否有资源泄漏风险
|
||
|
||
返回格式（JSON）：
|
||
```json
|
||
{{
|
||
"issues": [
|
||
{{
|
||
"severity": "critical|high|medium|low|info",
|
||
"line_number": 123,
|
||
"message": "问题描述",
|
||
"suggestion": "改进建议",
|
||
"score_penalty": 10.0
|
||
}}
|
||
]
|
||
}}
|
||
```
|
||
"""
|
||
|
||
try:
|
||
# 调用 LLM 进行深度审计
|
||
response = await self.llm._call_llm_chat(
|
||
messages=[
|
||
{"role": "system", "content": "你是一位严格但公平的代码审查专家。"},
|
||
{"role": "user", "content": prompt}
|
||
],
|
||
temperature=0.2,
|
||
)
|
||
|
||
# 解析 AI 响应
|
||
import json
|
||
json_match = re.search(r'```json\s*([\s\S]*?)\s*```', response)
|
||
if json_match:
|
||
ai_result = json.loads(json_match.group(1))
|
||
for issue_data in ai_result.get("issues", []):
|
||
# 限制 AI 生成的问题数量，避免过度敏感
|
||
if len(issues) < 10: # 最多 10 个 AI 发现的问题
|
||
severity_map = {
|
||
"critical": AuditSeverity.CRITICAL,
|
||
"high": AuditSeverity.HIGH,
|
||
"medium": AuditSeverity.MEDIUM,
|
||
"low": AuditSeverity.LOW,
|
||
"info": AuditSeverity.INFO,
|
||
}
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=severity_map.get(issue_data.get("severity", "info"), AuditSeverity.INFO),
|
||
category="functionality",
|
||
file_path=file_path,
|
||
line_number=issue_data.get("line_number"),
|
||
message=issue_data.get("message", ""),
|
||
suggestion=issue_data.get("suggestion", ""),
|
||
score_penalty=issue_data.get("score_penalty", 5.0),
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"AI functionality audit failed: {e}")
|
||
|
||
return issues
|
||
|
||
async def _ai_deep_audit(
|
||
self, task_name: str, commit_message: str, code_diff: str, changed_files: Dict[str, str]
|
||
) -> List[AuditIssue]:
|
||
"""AI 深度审计 - 跨文件分析"""
|
||
issues = []
|
||
|
||
# 限制文件数量，避免 prompt 过长
|
||
files_to_audit = list(changed_files.keys())[:5]
|
||
|
||
prompt = f"""作为一位资深架构师，请从系统层面审查以下代码变更：
|
||
|
||
任务：{task_name}
|
||
提交消息：{commit_message}
|
||
|
||
变更的文件：
|
||
{', '.join(files_to_audit)}
|
||
|
||
完整 diff：
|
||
```diff
|
||
{code_diff[:2000]} # 限制长度
|
||
```
|
||
|
||
请检查：
|
||
1. 架构一致性：变更是否符合项目架构
|
||
2. 依赖关系：是否引入了不兼容的依赖
|
||
3. 配置变更：是否需要更新配置文件
|
||
4. 迁移影响：是否需要数据迁移
|
||
5. 测试覆盖：是否需要添加测试
|
||
|
||
返回格式（JSON）：
|
||
```json
|
||
{{
|
||
"issues": [
|
||
{{
|
||
"severity": "critical|high|medium|low",
|
||
"category": "architecture|dependencies|configuration|migration|testing",
|
||
"message": "问题描述",
|
||
"suggestion": "建议",
|
||
"score_penalty": 5.0
|
||
}}
|
||
]
|
||
}}
|
||
```
|
||
"""
|
||
|
||
try:
|
||
response = await self.llm._call_llm_chat(
|
||
messages=[
|
||
{"role": "system", "content": "你是一位经验丰富的软件架构师。"},
|
||
{"role": "user", "content": prompt}
|
||
],
|
||
temperature=0.2,
|
||
)
|
||
|
||
import json
|
||
json_match = re.search(r'```json\s*([\s\S]*?)\s*```', response)
|
||
if json_match:
|
||
ai_result = json.loads(json_match.group(1))
|
||
for issue_data in ai_result.get("issues", []):
|
||
if len(issues) < 5: # 限制数量
|
||
severity_map = {
|
||
"critical": AuditSeverity.CRITICAL,
|
||
"high": AuditSeverity.HIGH,
|
||
"medium": AuditSeverity.MEDIUM,
|
||
"low": AuditSeverity.LOW,
|
||
}
|
||
issues.append(
|
||
AuditIssue(
|
||
severity=severity_map.get(issue_data.get("severity", "low"), AuditSeverity.LOW),
|
||
category=issue_data.get("category", "architecture"),
|
||
file_path="multiple",
|
||
message=issue_data.get("message", ""),
|
||
suggestion=issue_data.get("suggestion", ""),
|
||
score_penalty=issue_data.get("score_penalty", 3.0),
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"AI deep audit failed: {e}")
|
||
|
||
return issues
|
||
|
||
def _calculate_score(self, issues: List[AuditIssue]) -> float:
|
||
"""计算审计分数"""
|
||
# 初始满分
|
||
score = 100.0
|
||
|
||
# 按权重扣分
|
||
for issue in issues:
|
||
weight = self.WEIGHTS.get(issue.category, 10)
|
||
penalty = issue.score_penalty * (weight / 100)
|
||
score -= penalty
|
||
|
||
return max(0.0, score)
|
||
|
||
def _generate_summary(self, issues: List[AuditIssue], score: float) -> str:
|
||
"""生成审计摘要"""
|
||
if not issues:
|
||
return f"完美！代码审计通过，得分: {score:.1f}/100"
|
||
|
||
# 按严重程度统计
|
||
critical = sum(1 for i in issues if i.severity == AuditSeverity.CRITICAL)
|
||
high = sum(1 for i in issues if i.severity == AuditSeverity.HIGH)
|
||
medium = sum(1 for i in issues if i.severity == AuditSeverity.MEDIUM)
|
||
|
||
summary_parts = [f"审计得分: {score:.1f}/100"]
|
||
if critical > 0:
|
||
summary_parts.append(f"{critical} 个严重问题")
|
||
if high > 0:
|
||
summary_parts.append(f"{high} 个高优先级问题")
|
||
if medium > 0:
|
||
summary_parts.append(f"{medium} 个中等问题")
|
||
|
||
return " | ".join(summary_parts)
|
||
|
||
def _count_issues_by_severity(self, issues: List[AuditIssue]) -> Dict[str, int]:
|
||
"""按严重程度统计问题"""
|
||
counts = {s.value: 0 for s in AuditSeverity}
|
||
for issue in issues:
|
||
counts[issue.severity.value] += 1
|
||
return counts
|
||
|
||
def _count_issues_by_category(self, issues: List[AuditIssue]) -> Dict[str, int]:
|
||
"""按类别统计问题"""
|
||
counts = {}
|
||
for issue in issues:
|
||
counts[issue.category] = counts.get(issue.category, 0) + 1
|
||
return counts
|
||
|
||
def _detect_language(self, file_path: str) -> str:
|
||
"""检测文件语言"""
|
||
ext = file_path.rsplit(".", 1)[-1].lower()
|
||
lang_map = {
|
||
"py": "python",
|
||
"ts": "typescript",
|
||
"tsx": "typescript",
|
||
"js": "javascript",
|
||
"jsx": "javascript",
|
||
"vue": "vue",
|
||
"sql": "sql",
|
||
}
|
||
return lang_map.get(ext, "text")
|
||
|
||
|
||
# 全局审计器实例
|
||
_auditor = None
|
||
|
||
|
||
def get_auditor(llm_service) -> CodeAuditor:
|
||
"""获取代码审计器实例"""
|
||
global _auditor
|
||
if _auditor is None:
|
||
_auditor = CodeAuditor(llm_service)
|
||
return _auditor
|