from __future__ import annotations import os import pytest import pytest_asyncio from sqlalchemy import JSON from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine os.environ["DATABASE_URL"] = "sqlite+aiosqlite:///:memory:" from app.models.base import Base from app.models.reply_iteration import JudgeRun, QualityAsset, QualityAssetPair from app.models.training_pipeline import ( TrainedModelArtifact, TrainedModelDeployment, TrainingDataset, TrainingDatasetItem, TrainingEvaluation, TrainingJob, TrainingJobRun, ) from app.services.training_pipeline.service import TrainingPipelineService def _use_sqlite_json_types() -> None: for table in ( QualityAsset.__table__, QualityAssetPair.__table__, JudgeRun.__table__, TrainingDataset.__table__, TrainingDatasetItem.__table__, TrainingJob.__table__, TrainingJobRun.__table__, TrainingEvaluation.__table__, TrainedModelArtifact.__table__, TrainedModelDeployment.__table__, ): for column in table.columns: if column.name in { "source_snapshot", "tags", "selection_rules", "channel_scope", "redacted_snapshot", "metrics_json", "scope_json", "evaluation_summary", "effective_scope", }: column.type = JSON() @pytest_asyncio.fixture async def db_session(): engine = create_async_engine("sqlite+aiosqlite:///:memory:") _use_sqlite_json_types() async with engine.begin() as conn: await conn.run_sync( lambda sync_conn: Base.metadata.create_all( sync_conn, tables=[ QualityAsset.__table__, QualityAssetPair.__table__, JudgeRun.__table__, TrainingDataset.__table__, TrainingDatasetItem.__table__, TrainingJob.__table__, TrainingJobRun.__table__, TrainingEvaluation.__table__, TrainedModelArtifact.__table__, TrainedModelDeployment.__table__, ], ) ) maker = async_sessionmaker(engine, expire_on_commit=False) async with maker() as session: yield session await engine.dispose() @pytest.mark.asyncio async def test_training_dataset_rules_and_job_bootstrap(db_session) -> None: golden = QualityAsset( tenant_id=1, project_id=8, asset_type="golden", status="approved", prompt_key="PROMPT_FEISHU_DIRECT_QA", workflow_type="query.read_only", asked_via="feishu", question_text="现在系统走哪个层级", answer_text="当前走 high。", corrected_answer="当前走 high。", source_snapshot={"approved": True}, ) failure = QualityAsset( tenant_id=1, project_id=8, asset_type="failure", status="approved", prompt_key="PROMPT_FEISHU_DIRECT_QA", workflow_type="query.read_only", asked_via="feishu", question_text="这个回复为什么不对", answer_text="让我先看看。", corrected_answer="直接回答路由层级和原因。", source_snapshot={"approved": True}, ) db_session.add_all([golden, failure]) await db_session.flush() pair = QualityAssetPair( tenant_id=1, project_id=8, pair_type="preference", status="approved", chosen_asset_id=golden.id, rejected_asset_id=failure.id, reason_text="黄金样本明显更符合产品要求", ) db_session.add(pair) await db_session.commit() service = TrainingPipelineService(db_session) sft_dataset = await service.create_dataset( tenant_id=1, actor_id=9001, dataset_type="sft", version="sft-v1", asset_ids=[golden.id], pair_ids=[], selection_rules={"source": "manual"}, notes="SFT 首批数据集", ) preference_dataset = await service.create_dataset( tenant_id=1, actor_id=9001, dataset_type="preference", version="pref-v1", asset_ids=[], pair_ids=[pair.id], selection_rules={"source": "manual"}, notes="偏好对首批数据集", ) assert sft_dataset["dataset_type"] == "sft" assert sft_dataset["sample_count"] == 1 assert preference_dataset["dataset_type"] == "preference" assert preference_dataset["sample_count"] == 1 with pytest.raises(ValueError, match="SFT dataset only accepts approved golden assets"): await service.create_dataset( tenant_id=1, actor_id=9001, dataset_type="sft", version="sft-v2", asset_ids=[failure.id], pair_ids=[], selection_rules={"source": "manual"}, notes="invalid", ) job = await service.create_job( tenant_id=1, actor_id=9001, dataset_id=sft_dataset["id"], job_type="sft", base_model_family="glm", target_slot="high", provider_backend="openclaw-training", ) evaluations = await service.list_evaluations( tenant_id=1, job_id=job["id"], ) promoted = await service.promote_artifact( tenant_id=1, artifact_id=job["artifact"]["id"], actor_id=9001, ) decided = await service.decide_deployment( tenant_id=1, deployment_id=promoted["id"], actor_id=9001, decision="accepted", notes="评测通过，允许候选部署", ) assert job["status"] == "queued" assert job["artifact"]["target_slot"] == "high" assert {item["evaluation_type"] for item in evaluations["items"]} == { "offline_judge", "structure_compliance", "shadow_replay", } assert promoted["deployment_status"] == "candidate" assert decided["deployment_status"] == "active" run_result = await service.run_job( tenant_id=1, job_id=job["id"], actor_id=9001, ) assert run_result["job"]["status"] == "completed" assert run_result["run"]["status"] == "completed" assert {item["status"] for item in run_result["evaluations"]} == {"completed"} @pytest.mark.asyncio async def test_bootstrap_mounted_training_wave_creates_teacher_sets_and_jobs(db_session) -> None: golden_a = QualityAsset( tenant_id=1, project_id=8, asset_type="golden", status="approved", prompt_key="mounted.runtime.reply", workflow_type="mounted.plan", asked_via="feishu", question_text="这个需求先帮我梳理一下范围。", answer_text="我先帮你收敛目标、范围和负责人，再决定要不要出草稿。", corrected_answer="我先帮你收敛目标、范围和负责人，再决定要不要出草稿。", source_snapshot={"run_id": 11}, ) golden_b = QualityAsset( tenant_id=1, project_id=8, asset_type="golden", status="approved", prompt_key="mounted.runtime.reply", workflow_type="mounted.plan", asked_via="feishu", question_text="这件事先别执行，先做需求澄清。", answer_text="我先问一个关键问题：这次是先收敛范围，还是已经准备落草稿？", corrected_answer="我先问一个关键问题：这次是先收敛范围，还是已经准备落草稿？", source_snapshot={"run_id": 12}, ) failure = QualityAsset( tenant_id=1, project_id=8, asset_type="failure", status="approved", prompt_key="mounted.runtime.reply", workflow_type="mounted.execute", asked_via="feishu", question_text="那你觉得分配给谁做比较合适？", answer_text="我已经帮你执行分配。", corrected_answer="这更像负责人建议问题，我会先给候选人建议，不直接执行分配。", source_snapshot={"run_id": 13}, ) db_session.add_all([golden_a, golden_b, failure]) await db_session.flush() pair = QualityAssetPair( tenant_id=1, project_id=8, pair_type="preference", status="approved", chosen_asset_id=golden_a.id, rejected_asset_id=failure.id, reason_text="golden answer 更符合 mounted runtime 的 plan 模式。", ) db_session.add(pair) db_session.add( JudgeRun( tenant_id=1, project_id=8, asset_id=failure.id, prompt_key="mounted.runtime.reply", workflow_type="mounted.execute", asked_via="feishu", judge_status="completed", bucket_key="mounted_runtime.execute_safety", rule_score=72.0, llm_score=78.0, overall_score=75.0, findings=[{"code": "mis_execute", "severity": "critical"}], metadata_json={"run_id": 13}, ) ) await db_session.commit() service = TrainingPipelineService(db_session) wave = await service.bootstrap_mounted_training_wave( tenant_id=1, actor_id=9001, base_model_family="routing_default", provider_backend="openclaw-training", target_slots=["low", "high"], notes="mounted runtime training wave", ) dataset_roles = {item["role"] for item in wave["datasets"]} job_roles = {item["role"] for item in wave["jobs"]} assert dataset_roles == { "teacher_dev_set", "teacher_eval_frozen_set", "sft", "preference", } assert { "low_sft", "high_sft", "low_preference", "high_preference", "low_eval", "high_eval", }.issubset(job_roles) assert wave["acceptance_gate"]["ready_for_acceptance"] is False assert wave["acceptance_gate"]["blocking_reason"] == "frozen_eval_below_threshold" assert {item["status"] for item in wave["jobs"]} == {"completed"} assert all( row["run"]["status"] == "completed" for row in wave["executed_jobs"] )