llm 通过json列表判断

2025-06-19 18:33:03 +08:00
parent d3333d4f90
commit ba175cb2ae
32 changed files with 12241 additions and 8427 deletions
@@ -0,0 +1,9 @@
+[
+  "API接口应该遵循RESTful设计规范，URL应使用名词而非动词",
+  "API响应格式应统一，包含状态码、消息和数据三个字段",
+  "API应该妥善处理错误情况，返回适当的错误代码和说明",
+  "API应该使用正确的HTTP方法：GET用于检索，POST用于创建，PUT用于更新，DELETE用于删除",
+  "API响应中的时间字段应符合ISO 8601标准格式",
+  "API路径结构应遵循'<前缀>/<专业领域>/v<版本号>/<资源类型>'格式",
+  "API应提供适当的缓存控制机制"
+] 
@@ -0,0 +1,5 @@
+[
+
+  "API应该使用正确的HTTP方法：GET用于检索，POST用于创建，PUT用于更新，DELETE用于删除"
+
+] 
@@ -0,0 +1,108 @@
+import os
+import json
+from typing import Dict, Any, Optional, List
+from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
+
+class LLMComplianceCheckTestCase(BaseAPITestCase):
+    id = "TC-LLM-COMPLIANCE-001"
+    name = "LLM合规性综合检查"
+    description = "读取固定的合规性标准列表，将API所有关键信息（url、headers、params、query、body、示例响应等）发送给大模型，让其判断是否通过并给出理由。"
+    severity = TestSeverity.MEDIUM
+    tags = ["llm", "compliance", "auto-eval"]
+    execution_order = 99
+
+    def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
+        super().__init__(endpoint_spec, global_api_spec, json_schema_validator=json_schema_validator, llm_service=llm_service)
+        # 读取合规性标准
+        criteria_path = os.path.join(os.path.dirname(__file__), "compliance_criteria.json")
+        with open(criteria_path, "r", encoding="utf-8") as f:
+            self.compliance_criteria = json.load(f)
+        self.logger.info(f"已加载合规性标准: {self.compliance_criteria}")
+
+    def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
+        results = []
+        # 收集API所有关键信息
+        api_info = {
+            "method": request_context.method,
+            "url": request_context.url,
+            "path": self.endpoint_spec.get("path"),
+            "operationId": self.endpoint_spec.get("operationId"),
+            "headers": dict(request_context.headers) if hasattr(request_context, "headers") else {},
+            "query_params": getattr(request_context, "query_params", {}),
+            "path_params": getattr(request_context, "path_params", {}),
+            "body": getattr(request_context, "body", None),
+            "response_status": response_context.status_code,
+            "response_headers": dict(response_context.headers) if hasattr(response_context, "headers") else {},
+            "response_body": response_context.text_content if hasattr(response_context, "text_content") else None
+        }
+        # 日志打印所有API信息
+        self.logger.info("LLM合规性检查-API信息收集: " + json.dumps(api_info, ensure_ascii=False, indent=2))
+        self.logger.info("LLM合规性检查-标准: " + json.dumps(self.compliance_criteria, ensure_ascii=False, indent=2))
+
+        if not self.llm_service:
+            results.append(ValidationResult(
+                passed=True,
+                message="LLM服务不可用，跳过本用例。",
+                details={"reason": "llm_service is None"}
+            ))
+            return results
+
+        # 构建prompt
+        prompt = f"""
+你是一位API合规性专家。请根据以下合规性标准，对给定的API调用信息进行逐条评估。每条标准请给出是否通过（true/false）和理由。
+
+合规性标准:
+{json.dumps(self.compliance_criteria, ensure_ascii=False, indent=2)}
+
+API信息:
+{json.dumps(api_info, ensure_ascii=False, indent=2)}
+
+请以如下JSON格式输出：
+[
+  {{"criterion": "标准内容", "passed": true/false, "reason": "理由"}},
+  ...
+]
+"""
+        messages = [
+            {"role": "system", "content": "你是一位API合规性专家，输出必须是严格的JSON数组。"},
+            {"role": "user", "content": prompt}
+        ]
+        self.logger.info("发送给LLM的prompt: " + prompt)
+        llm_response_str = self.llm_service._execute_chat_completion_request(
+            messages=messages,
+            max_tokens=2048,
+            temperature=0.2
+        )
+        if not llm_response_str:
+            results.append(ValidationResult(
+                passed=False,
+                message="未能从LLM获取响应。",
+                details={"prompt": prompt}
+            ))
+            return results
+        self.logger.info(f"LLM原始响应: {llm_response_str}")
+        try:
+            cleaned = llm_response_str.strip()
+            if cleaned.startswith("```json"):
+                cleaned = cleaned[7:]
+            if cleaned.endswith("```"):
+                cleaned = cleaned[:-3]
+            llm_result = json.loads(cleaned)
+            if not isinstance(llm_result, list):
+                raise ValueError("LLM返回的不是JSON数组")
+            for item in llm_result:
+                criterion = item.get("criterion", "未知标准")
+                passed = item.get("passed", False)
+                reason = item.get("reason", "无理由")
+                results.append(ValidationResult(
+                    passed=passed,
+                    message=f"[{criterion}] {'通过' if passed else '不通过'}: {reason}",
+                    details={"criterion": criterion, "llm_reason": reason}
+                ))
+        except Exception as e:
+            results.append(ValidationResult(
+                passed=False,
+                message=f"LLM响应解析失败: {e}",
+                details={"raw_llm_response": llm_response_str}
+            ))
+        return results