This commit is contained in:
gongwenxin
2025-07-19 08:44:40 +08:00
parent 9a4afa8238
commit fcdfe71646
152 changed files with 598342 additions and 73747 deletions
@@ -0,0 +1 @@
# This file marks the compliance_catalog directory as a Python package.
@@ -0,0 +1 @@
# This file marks the core_functionality directory as a Python package.
@@ -0,0 +1,81 @@
from typing import Dict, Any, List, Optional
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext
from ddms_compliance_suite.utils.response_utils import extract_data_for_validation
from ddms_compliance_suite.utils.schema_provider import SchemaProvider
import logging
class FlexibleSchemaValidationCase(BaseAPITestCase):
"""
一个灵活的Schema验证测试用例,能够处理非标准的响应结构和动态的Schema来源。
"""
id = "TC-CORE-FUNC-002"
name = "灵活的返回体JSON Schema验证"
description = (
"验证API响应体是否符合预期的JSON Schema。此用例能够智能处理被包装的响应(如{code, data}),"
"并支持从列表响应中验证每个元素。它依赖于SchemaProvider获取schema,并设计为处理需要动态获取schema的场景。"
)
severity = TestSeverity.CRITICAL
tags = ["core-functionality", "schema-validation", "flexible"]
execution_order = 110 # 略高于标准Schema验证,以便在适用时优先执行
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service)
# We need to initialize the schema_provider here, as it's no longer injected.
self.schema_provider = SchemaProvider(global_api_spec) if global_api_spec else None
self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
def execute(self, request_context: APIRequestContext) -> List[ValidationResult]:
"""
执行灵活的schema验证。
"""
results = []
response_context = self.api_caller.call_api(request_context)
if not response_context:
return [self.failed("API调用失败,无法获取响应进行验证。")]
# 1. 使用 SchemaProvider 获取 Schema
if not self.schema_provider:
return [self.failed("SchemaProvider 未被初始化,无法执行此测试用例。")]
expected_schema = self.schema_provider.get_schema(self.endpoint_spec, response_context.status_code)
if not expected_schema:
# 如果是成功响应但找不到schema,这可能是一个问题
if 200 <= response_context.status_code < 300:
return [self.failed(f"成功响应(状态码 {response_context.status_code}),但无法为其找到或生成JSON Schema。")]
else:
return [self.passed(f"非成功响应(状态码 {response_context.status_code})且未定义Schema,跳过验证。")]
# 2. 使用 response_utils 提取待验证的数据列表
if not response_context.json_content:
return [self.failed(f"响应内容不是有效的JSON格式,无法进行Schema验证。响应文本: {response_context.text_content[:200]}...")]
data_to_validate_list = extract_data_for_validation(response_context.json_content)
if not data_to_validate_list:
# extract_data_for_validation 在找不到数据或遇到空列表时返回空列表
return [self.passed("未从响应中提取到需要验证的数据项(可能为空列表),跳过验证。")]
# 3. 遍历列表,对每个数据项进行验证
all_items_passed = True
for i, item in enumerate(data_to_validate_list):
item_context_prefix = f"响应列表中的第 {i+1} 个元素"
validation_results = self.validate_data_against_schema(
data_to_validate=item,
schema_definition=expected_schema,
context_message_prefix=item_context_prefix
)
for res in validation_results:
if not res.passed:
all_items_passed = False
# 为错误信息添加更多上下文
res.message = f"{item_context_prefix} {res.message}"
results.append(res)
if all_items_passed:
results.append(self.passed(f"成功验证了响应中的 {len(data_to_validate_list)} 个数据项,均符合Schema。"))
return results
@@ -0,0 +1,53 @@
from typing import Dict, Any, Optional, List
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import json
from ddms_compliance_suite.utils.schema_provider import SchemaProvider
class ResponseSchemaValidationCase(BaseAPITestCase):
id = "TC-CORE-FUNC-001"
name = "返回体JSON Schema验证"
description = "验证API响应体是否符合API规范中定义的JSON Schema。"
severity = TestSeverity.CRITICAL
tags = ["core-functionality", "schema-validation", "output-format"]
execution_order = 100 # Default, can be adjusted
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.schema_provider = SchemaProvider(global_api_spec) if global_api_spec else None
self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
results = []
method = request_context.method.upper()
status_code = response_context.status_code
if not self.schema_provider:
return [self.failed("SchemaProvider 未被初始化,无法执行此测试用例。")]
expected_schema = self.schema_provider.get_schema(self.endpoint_spec, status_code)
if expected_schema and response_context.json_content is not None:
self.logger.info(f"将根据从API规范中获取的schema验证响应体。")
schema_validation_results = self.validate_data_against_schema(
data_to_validate=response_context.json_content,
schema_definition=expected_schema,
context_message_prefix=f"针对 {method} {request_context.url} (状态码 {status_code}) 的响应体"
)
results.extend(schema_validation_results)
elif response_context.json_content is None and method not in ["DELETE", "HEAD", "OPTIONS"] and status_code in [200, 201, 202]:
if expected_schema:
results.append(self.failed(
message=f"根据schema期望一个JSON响应体,但未收到可解析的JSON内容。",
details={"status_code": status_code, "response_text_sample": (response_context.text_content or "")[:200]}
))
self.logger.warning(f"期望 {method} {request_context.url} 返回JSON响应体,但未收到或非JSON格式。")
elif not expected_schema and response_context.json_content is not None and status_code // 100 == 2:
self.logger.info(f"响应包含JSON体,但在API规范中未找到针对状态码 {status_code} 的JSON schema。跳过schema验证。")
elif not expected_schema and response_context.json_content is None:
self.logger.info(f"状态码 {status_code} 的响应无JSON体也无定义的schema。跳过schema验证。")
if not results:
results.append(self.passed("标准Schema验证步骤完成(未发现问题,或schema不适用/未为此响应定义)。"))
return results
@@ -0,0 +1 @@
# This file marks the error_handling directory as a Python package.
@@ -0,0 +1,196 @@
from typing import Dict, Any, Optional, List, Union, Tuple
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import logging
from ddms_compliance_suite.utils import schema_utils
import random
import string
class InvalidEnumValueCase(BaseAPITestCase):
id = "TC-ERROR-4006"
name = "非法枚举值检查"
description = "测试当发送的参数值不在指定的枚举范围内时,API是否按预期返回code=4006的错误。"
severity = TestSeverity.MEDIUM
tags = ["error-handling", "appendix-b", "4006", "invalid-enum"]
execution_order = 204 # 在数值越界之后执行
@staticmethod
def _endpoint_has_enum_field(endpoint_spec: Dict[str, Any], logger: logging.Logger) -> bool:
"""
静态辅助方法,检查端点规范中是否有任何字段(body或parameters)包含枚举定义。
"""
# 1. 检查请求体 (Body)
if endpoint_spec.get("requestBody"):
# 注意:这里需要一个可以解析$ref的上下文,但applies_to是静态的。
# 这是一个简化检查,它只检查顶级schema。
# 更完整的检查需要一个能够解析$ref的schema工具。
# 为了简单起见,我们假设测试用例的 _get_resolved_request_body_schema 可以在实例化后使用。
# 这里的检查可能不完整,但可以覆盖大部分情况。
if schema_utils.util_find_enum_field_recursive(endpoint_spec["requestBody"].get("content", {}).get("application/json", {}).get("schema", {}), [], logger):
return True
# 2. 检查查询参数, 路径参数, 请求头
if 'parameters' in endpoint_spec:
for param in endpoint_spec['parameters']:
param_desc = param.get('description', '')
if schema_utils.util_extract_enum_from_description(param_desc, logger):
return True
return False
@classmethod
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
"""
此测试用例仅适用于那些在请求体或参数中定义了枚举(enum)的端点。
"""
# 创建一个临时logger用于静态方法
temp_logger = logging.getLogger(f"applies_to.{cls.id}")
return cls._endpoint_has_enum_field(endpoint_spec, temp_logger)
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.setLevel(logging.INFO)
self.target_field_location: Optional[str] = None
self.target_field_path: Optional[List[Union[str, int]]] = None
self.target_field_name: Optional[str] = None
self.valid_enums: Optional[List[str]] = None
self.invalid_enum_value: Optional[str] = None
self._initialize_test_case()
def _find_first_enum_field(self) -> Optional[Tuple[str, List[Union[str, int]], str, List[str], str]]:
"""
按优先级(body -> query -> path -> header)查找第一个具有枚举值限制的字段。
"""
self.logger.info(f"[{self.id}] Searching for a field with enum constraints...")
# 1. 检查请求体 (Body)
body_schema = self._get_resolved_request_body_schema()
if body_schema:
self.logger.debug(f"[{self.id}] Checking for enum field in request body.")
result = schema_utils.util_find_enum_field_recursive(body_schema, [], self.logger)
if result:
path, valid_enums, desc = result
name = path[-1] if path else "body_root"
self.logger.info(f"[{self.id}] Found enum field in BODY at path: {'.'.join(map(str, path))}")
return "body", path, name, valid_enums, desc
# 2. 检查查询参数, 路径参数, 请求头
for location in ["query", "path", "header"]:
if 'parameters' in self.endpoint_spec:
self.logger.debug(f"[{self.id}] Checking for enum field in {location} parameters.")
for param in self.endpoint_spec['parameters']:
if param.get('in') == location:
param_desc = param.get('description', '')
param_name = param.get('name')
valid_enums = schema_utils.util_extract_enum_from_description(param_desc, self.logger)
if valid_enums:
self.logger.info(f"[{self.id}] Found enum field in {location.upper()}: '{param_name}'")
return location, [param_name], param_name, valid_enums, param_desc
self.logger.info(f"[{self.id}] No field with enum constraints found in any location.")
return None
def _initialize_test_case(self):
"""初始化测试用例。"""
found_field = self._find_first_enum_field()
if found_field:
location, path, name, valid_enums, description = found_field
self.target_field_location = location
self.target_field_path = path
self.target_field_name = name
self.valid_enums = valid_enums
self.logger.info(
f"[{self.id}] Target found in '{location}' at path '{'.'.join(map(str, path))}'. "
f"Valid enums: {valid_enums}, Desc: '{description}'"
)
self._set_invalid_enum_value()
else:
self.logger.info(f"[{self.id}] No suitable enum field found for this endpoint.")
def _set_invalid_enum_value(self):
"""生成一个非法的枚举值。"""
random_suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=6))
self.invalid_enum_value = f"invalid_enum_{random_suffix}"
self.logger.info(f"[{self.id}] Setting invalid enum value to '{self.invalid_enum_value}' (valid are {self.valid_enums}).")
def _modify_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
"""通用函数,用于修改参数字典中的值。"""
if self.invalid_enum_value is None or not self.target_field_path:
return params
modified_params, success = schema_utils.util_set_value_at_path(
data_container=params,
path=self.target_field_path,
new_value=self.invalid_enum_value
)
if success:
self.logger.debug(f"[{self.id}] Successfully injected invalid enum value into parameters.")
return modified_params
self.logger.error(f"[{self.id}] Failed to set invalid enum value in parameters.")
return params
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "query":
return self._modify_params(current_query_params)
return current_query_params
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "path":
return self._modify_params(current_path_params)
return current_path_params
def generate_headers(self, current_headers: Dict[str, str]) -> Dict[str, str]:
if self.target_field_location == "header":
return self._modify_params(current_headers)
return current_headers
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
if self.target_field_location == "body":
return self._modify_params(current_body)
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4006错误(状态码为200)。"""
results = []
if self.invalid_enum_value is None:
return [self.passed("跳过测试:未找到具有明确枚举值限制的字段。")]
expected_http_status_code = 200
expected_business_error_code = -1
context_msg_prefix = (
f"{self.target_field_location} 字段 '{self.target_field_name}' "
f"值为 '{self.invalid_enum_value}' (合法值为: {self.valid_enums}) 时, "
)
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if actual_business_code != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}"
)]
@@ -0,0 +1,98 @@
from typing import Dict, Any, Optional, List, Union
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import logging
from ddms_compliance_suite.utils import schema_utils # Keep this import for util_remove_value_at_path
class MissingRequiredFieldBodyCase(BaseAPITestCase):
id = "TC-ERROR-4003-BODY"
name = "缺失必填请求体字段检查"
description = "测试当请求体中缺少API规范定义的必填字段时,API是否按预期返回code=4003的错误。"
severity = TestSeverity.HIGH
tags = ["error-handling", "appendix-b", "4003", "required-fields", "request-body"]
execution_order = 210
@classmethod
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
"""
此测试用例仅适用于那些在API规范中明确定义了请求体(requestBody)的端点。
"""
# 如果 'requestBody' 存在且其内容不为空,则认为此测试用例适用。
return bool(endpoint_spec.get("requestBody"))
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service)
self.logger = logging.getLogger(f"testcase.{self.id}") # Already set in super, but can be re-set if specific sub-logger is needed. Better to rely on super's logger.
self.original_value_at_path: Any = None
# Use new helper methods from BaseAPITestCase
body_schema = self._get_resolved_request_body_schema()
if body_schema:
self.removed_field_path = self._find_removable_field_path(
schema_to_search=body_schema,
schema_name_for_log="request body"
)
if not self.removed_field_path:
self.logger.info('在请求体 schema 中未找到可用于测试 "必填字段缺失" 的字段(通过基类方法)。')
else:
self.logger.info('此端点规范中未定义或找到请求体 schema(通过基类方法)。')
self.removed_field_path = None
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
self.logger.debug(f"{self.id} is focused on request body, generate_query_params will not modify query parameters.")
return current_query_params
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
if not self.removed_field_path:
self.logger.debug("No field path identified for removal in request body. Returning original body.")
return current_body
modified_body, self.original_value_at_path, success = schema_utils.util_remove_value_at_path(
data_container=current_body,
path=self.removed_field_path
)
if success:
self.logger.info(f"为进行必填字段缺失测试,已通过工具方法从请求体中移除字段路径 '{'.'.join(map(str, self.removed_field_path))}'")
return modified_body
else:
self.logger.error(f"使用工具方法移除请求体字段路径 '{'.'.join(map(str, self.removed_field_path))}' 失败。将返回原始请求体。")
# Restore original_value_at_path to None since removal failed
self.original_value_at_path = None
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4003错误(状态码为200)。"""
results = []
if not self.removed_field_path:
results.append(self.passed("跳过测试:在API规范中未找到合适的必填请求体字段用于移除测试。"))
self.logger.info("由于未识别到可移除的必填请求体字段,跳过此测试用例的验证。")
return results
expected_http_status_code = 200
expected_business_error_code = -1
removed_field_str = '.'.join(map(str, self.removed_field_path))
context_msg_prefix = f"当移除必填请求体字段 '{removed_field_str}' 时, "
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if actual_business_code != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}"
)]
@@ -0,0 +1,92 @@
from typing import Dict, Any, Optional, List
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import copy
class MissingRequiredFieldQueryCase(BaseAPITestCase):
id = "TC-ERROR-4003-QUERY"
name = "缺失必填查询参数检查"
description = "测试当请求中缺少API规范定义的必填查询参数时,API是否按预期返回code=4003的错误。"
severity = TestSeverity.HIGH
tags = ["error-handling", "appendix-b", "4003", "required-fields", "query-parameters"]
execution_order = 211 # After body, before original combined one might have been
@classmethod
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
"""
此测试用例仅适用于那些在API规范中定义了查询参数(parameters with 'in' == 'query')的端点。
"""
# 如果 'parameters' 字段不存在,则不适用
if not endpoint_spec.get("parameters"):
return False
# 遍历所有参数,如果发现任何一个参数的 "in" 字段是 "query",则此测试用例适用。
return any(param.get("in") == "query" for param in endpoint_spec["parameters"])
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.target_param_name: Optional[str] = None
# Location is always 'query' for this class
self.target_param_location: str = "query"
self.original_query_params: Optional[Dict[str, Any]] = None
# Call the simplified method to find the target parameter
self._try_find_removable_query_param()
self.logger.info(f"测试用例 {self.id} ({self.name}) 已针对端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。Target param to remove: {self.target_param_name}")
def _try_find_removable_query_param(self):
"""Uses the base class helper to find a required query parameter."""
self.target_param_name = self._find_required_parameter_name("query")
# Logging about success/failure is handled by the base class method and the __init__ method.
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
# This test case focuses on query parameters, so it does not modify the request body.
self.logger.debug(f"{self.id} is focused on query parameters, generate_request_body will not modify the request body.")
return current_body
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_param_name and isinstance(current_query_params, dict):
if self.target_param_name in current_query_params:
new_params = copy.deepcopy(current_query_params)
original_value = new_params.pop(self.target_param_name) # 移除参数
self.logger.info(f"为进行必填查询参数缺失测试,已从查询参数中移除 '{self.target_param_name}' (原值: '{original_value}')。")
return new_params
else:
self.logger.warning(f"计划移除的查询参数 '{self.target_param_name}' 在当前查询参数中未找到。")
return current_query_params
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4003错误(状态码为200)。"""
results = []
if not self.target_param_name:
results.append(self.passed("跳过测试:在API规范中未找到合适的必填查询参数用于移除测试。"))
self.logger.info("由于未识别到可移除的必填查询参数,跳过此测试用例。")
return results
expected_http_status_code = 200
expected_business_error_code = -1
context_msg_prefix = f"当移除必填查询参数 '{self.target_param_name}' 时, "
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if actual_business_code != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}"
)]
@@ -0,0 +1,227 @@
from typing import Dict, Any, Optional, List, Union, Tuple
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import logging
from ddms_compliance_suite.utils import schema_utils
class NumberOutOfRangeCase(BaseAPITestCase):
id = "TC-ERROR-4002"
name = "数值参数越界检查"
description = "测试当发送的数值参数超出范围限制时,API是否按预期返回code=4002的错误。"
severity = TestSeverity.MEDIUM
tags = ["error-handling", "appendix-b", "4002", "out-of-range"]
execution_order = 203 # 在类型不匹配测试之后执行
@staticmethod
def _endpoint_has_ranged_field(endpoint_spec: Dict[str, Any], logger: logging.Logger) -> bool:
"""
静态辅助方法,检查端点规范中是否有任何字段(body或parameters)包含范围定义。
"""
# 1. 检查请求体 (Body)
if endpoint_spec.get("requestBody"):
if schema_utils.util_find_ranged_field_recursive(endpoint_spec["requestBody"].get("content", {}).get("application/json", {}).get("schema", {}), [], logger):
return True
# 2. 检查查询参数, 路径参数, 请求头
if 'parameters' in endpoint_spec:
for param in endpoint_spec['parameters']:
param_schema = param.get('schema', {})
param_desc = param.get('description', '')
min_val, max_val = schema_utils.util_extract_range_from_description(param_desc, logger)
if min_val is None and max_val is None:
min_val = param_schema.get('minimum')
max_val = param_schema.get('maximum')
if min_val is not None or max_val is not None:
return True
return False
@classmethod
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
"""
此测试用例仅适用于那些在请求体或参数中定义了数值范围的端点。
"""
temp_logger = logging.getLogger(f"applies_to.{cls.id}")
return cls._endpoint_has_ranged_field(endpoint_spec, temp_logger)
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.setLevel(logging.DEBUG)
self.target_field_location: Optional[str] = None
self.target_field_path: Optional[List[Union[str, int]]] = None
self.target_field_schema: Optional[Dict[str, Any]] = None
self.target_field_name: Optional[str] = None
self.target_field_type: Optional[str] = None
self.min_value: Optional[float] = None
self.max_value: Optional[float] = None
self.out_of_range_value: Optional[Union[int, float]] = None
self._initialize_test_case()
def _find_first_ranged_numeric_field(self) -> Optional[Tuple[str, List[Union[str, int]], Dict[str, Any], str, Optional[str], Optional[float], Optional[float], str]]:
"""
按优先级(body -> query -> path -> header)查找第一个具有范围限制的数值字段。
"""
self.logger.info(f"[{self.id}] Searching for a ranged numeric field...")
# 1. 检查请求体 (Body)
body_schema = self._get_resolved_request_body_schema()
if body_schema:
self.logger.debug(f"[{self.id}] Checking for ranged numeric field in request body.")
result = schema_utils.util_find_ranged_field_recursive(body_schema, [], self.logger)
if result:
path, schema, type, min_val, max_val, desc = result
name = path[-1] if path else "body_root"
self.logger.info(f"[{self.id}] Found ranged numeric field in BODY at path: {'.'.join(map(str, path))}")
return "body", path, schema, name, type, min_val, max_val, desc
# 2. 检查查询参数, 路径参数, 请求头
for location in ["query", "path", "header"]:
if 'parameters' in self.endpoint_spec:
self.logger.debug(f"[{self.id}] Checking for ranged numeric field in {location} parameters.")
for param in self.endpoint_spec['parameters']:
if param.get('in') == location:
param_schema = param.get('schema', {})
param_desc = param.get('description', '')
param_name = param.get('name')
min_val, max_val = schema_utils.util_extract_range_from_description(param_desc, self.logger)
if min_val is None and max_val is None:
min_val = param_schema.get('minimum')
max_val = param_schema.get('maximum')
if min_val is not None or max_val is not None:
param_type = param_schema.get('type')
self.logger.info(f"[{self.id}] Found ranged numeric field in {location.upper()}: '{param_name}'")
return location, [param_name], param_schema, param_name, param_type, min_val, max_val, param_desc
self.logger.info(f"[{self.id}] No ranged numeric field found in any location.")
return None
def _initialize_test_case(self):
"""使用框架辅助方法初始化测试用例。"""
found_field = self._find_first_ranged_numeric_field()
if found_field:
location, path, schema, name, field_type, min_val, max_val, description = found_field
self.target_field_location = location
self.target_field_path = path
self.target_field_schema = schema
self.target_field_name = name
# 如果'type'未定义,我们根据是否存在小数来推断
self.target_field_type = field_type if field_type else ('number' if isinstance(min_val, float) or isinstance(max_val, float) else 'integer')
self.min_value = min_val
self.max_value = max_val
self.logger.info(
f"[{self.id}] Target found in '{location}' at path '{'.'.join(map(str, path))}'. "
f"Type: {self.target_field_type}, Min: {min_val}, Max: {max_val}, Desc: '{description}'"
)
self._set_out_of_range_value()
else:
self.logger.info(f"[{self.id}] No suitable ranged numeric field found for this endpoint.")
def _set_out_of_range_value(self):
"""根据提取的范围信息设置越界值。"""
# 优先选择超过最大值
if self.max_value is not None:
if self.target_field_type == 'integer':
self.out_of_range_value = int(self.max_value) + 1
else:
self.out_of_range_value = self.max_value + 1.0
self.logger.info(f"[{self.id}] Setting out-of-range value to {self.out_of_range_value} (max is {self.max_value}).")
return
# 其次选择小于最小值
if self.min_value is not None:
if self.target_field_type == 'integer':
self.out_of_range_value = int(self.min_value) - 1
else:
self.out_of_range_value = self.min_value - 1.0
self.logger.info(f"[{self.id}] Setting out-of-range value to {self.out_of_range_value} (min is {self.min_value}).")
def _modify_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
"""通用函数,用于修改参数字典中的值。"""
if self.out_of_range_value is None or not self.target_field_path:
return params
modified_params, success = schema_utils.util_set_value_at_path(
data_container=params,
path=self.target_field_path,
new_value=self.out_of_range_value
)
if success:
self.logger.debug(f"[{self.id}] Successfully injected out-of-range value into parameters.")
return modified_params
self.logger.error(f"[{self.id}] Failed to set out-of-range value in parameters.")
return params
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "query":
return self._modify_params(current_query_params)
return current_query_params
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "path":
return self._modify_params(current_path_params)
return current_path_params
def generate_headers(self, current_headers: Dict[str, str]) -> Dict[str, str]:
if self.target_field_location == "header":
# Header values must be strings
original_value = self.out_of_range_value
self.out_of_range_value = str(self.out_of_range_value)
headers = self._modify_params(current_headers)
self.out_of_range_value = original_value # revert for logging
return headers
return current_headers
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
if self.target_field_location == "body":
return self._modify_params(current_body)
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4002错误(状态码为200)。"""
results = []
if self.out_of_range_value is None:
return [self.passed("跳过测试:未找到具有明确范围限制的数值字段。")]
expected_http_status_code = 200
expected_business_error_code = -1
context_msg_prefix = (
f"{self.target_field_location} 字段 '{self.target_field_name}' "
f"值为 {self.out_of_range_value} (超出范围: min={self.min_value}, max={self.max_value}) 时, "
)
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if actual_business_code != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}"
)]
@@ -0,0 +1,125 @@
from typing import Dict, Any, Optional, List, Union
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import copy
import logging
from ddms_compliance_suite.utils import schema_utils
class TypeMismatchBodyCase(BaseAPITestCase):
id = "TC-ERROR-4001-BODY"
name = "请求体字段类型不匹配检查"
description = "测试当发送的请求体中字段的数据类型与API规范定义不符时,API是否按预期返回code=4001的错误。"
severity = TestSeverity.MEDIUM
tags = ["error-handling", "appendix-b", "4001", "request-body"]
execution_order = 202 # Slightly after query param one
@classmethod
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
"""
此测试用例仅适用于那些在API规范中明确定义了请求体(requestBody)的端点。
"""
# 如果 'requestBody' 存在且其内容不为空,则认为此测试用例适用。
return bool(endpoint_spec.get("requestBody"))
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.setLevel(logging.DEBUG)
self.target_field_path: Optional[List[str]] = None
self.original_field_type: Optional[str] = None
# Location is always 'body' for this class
self.target_field_location: str = "body"
self.target_field_schema: Optional[Dict[str, Any]] = None
self.json_schema_validator = json_schema_validator
self.original_value_at_path: Any = None
self.mismatched_value: Any = None
self._try_find_mismatch_target_in_body()
def _try_find_mismatch_target_in_body(self):
self.logger.info(f"[{self.id}] Initializing: Looking for a simple type field in request body for type mismatch test.")
body_schema_to_check = self._get_resolved_request_body_schema()
if body_schema_to_check:
found_target = self._find_simple_type_field_in_schema(body_schema_to_check, "request body")
if found_target:
self.target_field_path, self.original_field_type, self.target_field_schema = found_target
self.logger.info(f"[{self.id}] Target field for type mismatch (body): {'.'.join(map(str, self.target_field_path))}, Original Type: {self.original_field_type}")
else:
self.logger.info(f"[{self.id}] No suitable simple type field found in request body schema for type mismatch test.")
else:
self.logger.info(f"[{self.id}] No request body schema found for endpoint. Skipping type mismatch target search.")
if not self.target_field_path:
self.logger.info(f"[{self.id}] Conclusion: No target field identified for request body type mismatch test.")
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
self.logger.debug(f"{self.id} is focused on request body, generate_query_params will not modify query parameters.")
return current_query_params
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
if not self.target_field_path or not self.original_field_type:
self.logger.info(f"[{self.id}] No target field or original type identified for body type mismatch. Skipping body modification.")
return current_body
self.logger.debug(f"[{self.id}] Preparing to modify request body for type mismatch. Target path: {self.target_field_path}, Original type: {self.original_field_type}")
# Get the original value at path for logging/context if needed (optional)
# current_val_at_path, _ = schema_utils.util_get_value_at_path(current_body, self.target_field_path) # Assuming a get_value_at_path util exists or is added
# For now, we don't strictly need original_value for generate_mismatched_value, but it takes it as an arg.
mismatched_value = schema_utils.generate_mismatched_value(
original_type=self.original_field_type,
original_value=None, # Placeholder, as current generate_mismatched_value doesn't use it heavily yet
field_schema=self.target_field_schema,
logger_param=self.logger
)
self.logger.info(f"[{self.id}] Generated mismatched value '{mismatched_value}' for original type '{self.original_field_type}' at path '{'.'.join(map(str, self.target_field_path))}'.")
modified_body, success = schema_utils.util_set_value_at_path(
data_container=current_body,
path=self.target_field_path,
new_value=mismatched_value
)
if success:
self.logger.debug(f"[{self.id}] Successfully set mismatched value at path using util_set_value_at_path.")
return modified_body
else:
self.logger.error(f"[{self.id}] Failed to set mismatched value at path using util_set_value_at_path. Returning original body.")
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4001错误(状态码为200)。"""
results = []
if not self.target_field_path:
self.logger.info(f"[{self.id}] Skipped type mismatch (body) validation: No target field was identified.")
return [self.passed("跳过测试:在请求体中未找到合适的字段来测试类型不匹配。")]
expected_http_status_code = 200
expected_business_error_code = -1
field_path_str = '.'.join(map(str, self.target_field_path))
context_msg_prefix = f"当请求体字段 '{field_path_str}' 类型不匹配时, "
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if actual_business_code != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}"
)]
@@ -0,0 +1,132 @@
from typing import Dict, Any, Optional, List, Tuple, Union
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import copy
import logging
from ddms_compliance_suite.utils import schema_utils
class TypeMismatchQueryParamCase(BaseAPITestCase):
id = "TC-ERROR-4001-QUERY"
name = "查询参数类型不匹配检查"
description = "测试当发送的查询参数数据类型与API规范定义不符时,API是否按预期返回code=4001的错误。"
severity = TestSeverity.MEDIUM
tags = ["error-handling", "appendix-b", "4001", "query-parameters"]
execution_order = 201 # Slightly after the combined one might have been
@classmethod
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
"""
此测试用例仅适用于那些在API规范中定义了查询参数(parameters with 'in' == 'query')的端点。
"""
# 如果 'parameters' 字段不存在,则不适用
if not endpoint_spec.get("parameters"):
return False
# 遍历所有参数,如果发现任何一个参数的 "in" 字段是 "query",则此测试用例适用。
return any(param.get("in") == "query" for param in endpoint_spec["parameters"])
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.setLevel(logging.DEBUG)
self.target_field_path: Optional[List[str]] = None
self.original_field_type: Optional[str] = None
# Location is always 'query' for this class
self.target_field_location: str = "query"
self.target_field_schema: Optional[Dict[str, Any]] = None
self.target_param_name: Optional[str] = None
self.json_schema_validator = json_schema_validator
self.original_value_at_path: Any = None
self.mismatched_value: Any = None
# 调用新方法来查找目标字段
self._try_find_mismatch_target_in_query()
def _try_find_mismatch_target_in_query(self):
self.logger.info(f"[{self.id}] Initializing: Looking for a simple type query parameter for type mismatch test.")
found_target_param = self._find_first_simple_type_parameter(param_location="query")
if found_target_param:
full_path, param_type, param_schema, top_level_param_name = found_target_param
self.target_field_path = full_path
self.original_field_type = param_type
self.target_field_schema = param_schema
self.target_param_name = top_level_param_name # Store the top-level parameter name
self.logger.info(f"[{self.id}] Target for type mismatch (query): Param='{self.target_param_name}', Path='{'.'.join(map(str,self.target_field_path))}', Type='{self.original_field_type}'")
else:
self.logger.info(f"[{self.id}] No suitable simple type query parameter found for type mismatch test.")
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
self.logger.debug(f"{self.id} is focused on query parameters, generate_request_body will not modify the body.")
return current_body
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
if not self.target_field_path or not self.original_field_type:
self.logger.info(f"[{self.id}] No target field or original type identified for query param type mismatch. Skipping query param modification.")
return current_query_params
self.logger.debug(f"[{self.id}] Preparing to modify query params for type mismatch. Target path: {self.target_field_path}, Original type: {self.original_field_type}")
mismatched_value = schema_utils.generate_mismatched_value(
original_type=self.original_field_type,
original_value=None, # Placeholder
field_schema=self.target_field_schema,
logger_param=self.logger
)
self.logger.info(f"[{self.id}] Generated mismatched value '{mismatched_value}' for original type '{self.original_field_type}' at query path '{'.'.join(map(str, self.target_field_path))}'.")
# Query parameters are typically a flat dictionary, but util_set_value_at_path can handle nested paths if needed (e.g. for object-style query params)
modified_params, success = schema_utils.util_set_value_at_path(
data_container=current_query_params,
path=self.target_field_path, # Path might be like ['paramName'] or ['paramName', 'nestedKey']
new_value=mismatched_value
)
if success:
self.logger.debug(f"[{self.id}] Successfully set mismatched value in query params using util_set_value_at_path.")
return modified_params
else:
self.logger.error(f"[{self.id}] Failed to set mismatched value in query params using util_set_value_at_path. Returning original params.")
return current_query_params
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4001错误(状态码为200)。"""
results = []
if not self.target_field_path:
self.logger.info(f"[{self.id}] Skipped type mismatch (query) validation: No target query parameter was identified.")
return [self.passed("跳过测试:在查询参数中未找到合适的字段来测试类型不匹配。")]
expected_http_status_code = 200
expected_business_error_code = -1
context_param_identifier = self.target_param_name or '.'.join(map(str, self.target_field_path))
context_msg_prefix = f"当查询参数 '{context_param_identifier}' (路径: '{'.'.join(map(str, self.target_field_path))}') 类型不匹配时, "
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if actual_business_code != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}"
)]
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
# ... existing code ...
pass
return current_path_params
@@ -0,0 +1 @@
# This file marks the normative_spec directory as a Python package.
@@ -0,0 +1,67 @@
from typing import Dict, Any, Optional, List
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
# class HTTPMethodUsageCase(BaseAPITestCase):
# id = "TC-NORMATIVE-001"
# name = "HTTP Method Usage Verification"
# description = "验证API是否恰当使用HTTP方法(例如,GET用于检索,POST用于创建)。目前不测试对不支持方法的405响应。"
# severity = TestSeverity.MEDIUM
# tags = ["normative-spec", "http", "restful"]
# execution_order = 110
# # 此测试通常适用。
# # 检查对不支持方法的405响应会比较复杂,需要知道哪些方法对于每个路径是明确不支持的,
# # 或者尝试所有其他方法,这在API规范中并不总是明确的。
# def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None):
# super().__init__(endpoint_spec, global_api_spec, json_schema_validator)
# self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
# def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
# results = []
# method = request_context.method.upper()
# status_code = response_context.status_code
# # 基于常见RESTful约定的基本检查
# # 这些是通用指南,可能需要根据具体的API设计进行调整。
# if method == "GET":
# if status_code // 100 == 2: # 成功的GET
# results.append(self.passed(f"GET请求 {request_context.url} 返回了成功的状态码 {status_code}。"))
# elif status_code == 404:
# results.append(self.passed(f"GET请求 {request_context.url} 返回404,如果资源不存在则这是有效的响应。"))
# # GET请求的其他状态码可能是错误或此处未覆盖的特定条件。
# elif method == "POST":
# if status_code == 201: # 已创建
# results.append(self.passed(f"POST请求 {request_context.url} 返回201 Created,符合资源创建的预期。"))
# elif status_code == 200 or status_code == 202: # OK或已接受(例如,用于异步任务)
# results.append(self.passed(f"POST请求 {request_context.url} 返回{status_code},这可以是有效的响应。"))
# # 可添加对400(错误请求,例如payload无效)等的检查。
# elif method == "PUT":
# if status_code == 200: # OK(已更新)
# results.append(self.passed(f"PUT请求 {request_context.url} 返回200 OK,符合资源更新的预期。"))
# elif status_code == 201: # 已创建(如果PUT在资源不存在时创建资源)
# results.append(self.passed(f"PUT请求 {request_context.url} 返回201 Created,这可以是有效的响应。"))
# elif status_code == 204: # 无内容(已更新,不返回响应体)
# results.append(self.passed(f"PUT请求 {request_context.url} 返回204 No Content,这可以是有效的响应。"))
# # 可添加对404(未找到,如果要更新的资源不存在,除非PUT会创建)的检查。
# elif method == "DELETE":
# if status_code == 200 or status_code == 202 or status_code == 204: # OK、已接受或无内容
# results.append(self.passed(f"DELETE请求 {request_context.url} 返回{status_code},表示成功删除。"))
# # 可添加对404(未找到,如果要删除的资源不存在)的检查。
# # 405(方法不允许)检查的占位符 - 这比较复杂
# # 要测试405,通常需要:
# # 1. 知道哪些方法是此路径明确不允许的。
# # 2. 或者,尝试使用其他常见方法(OPTIONS, PATCH等)发送请求,
# # 如果这些方法未在此路径的规范中定义,则期望405。
# # 这通常需要更具针对性的测试用例或不同的方法。
# # self.logger.info("此通用测试用例未实现405(方法不允许)检查。")
# if not results: # 如果没有为该方法触发特定的验证
# results.append(self.passed(f"针对 {method} {request_context.url} 的HTTP方法使用检查完成(基于通用约定未发现特定问题)。"))
# return results
@@ -0,0 +1,247 @@
# import re
# import json # 确保导入json
# from typing import Dict, Any, Optional, List
# from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext
# # LLMService的导入路径需要根据您的项目结构确认
# # 假设 LLMService 在 ddms_compliance_suite.llm_utils.llm_service
# try:
# from ddms_compliance_suite.llm_utils.llm_service import LLMService
# except ImportError:
# LLMService = None
# # print("LLMService not found, PathVerbNounCheckCase will be skipped or limited.")
# class ComprehensiveURLCheckLLMCase(BaseAPITestCase):
# id = "TC-NORMATIVE-URL-LLM-COMPREHENSIVE-001"
# name = "综合URL规范与RESTful风格检查 (LLM)"
# description = (
# "使用LLM统一评估API路径是否符合以下规范:\n"
# "1. 路径参数命名 (例如,全小写蛇形命名法)。\n"
# "2. URL路径结构 (例如,/{领域}/{版本号}/资源类型)。\n"
# "3. URL版本号嵌入 (例如,包含 /v1/)。\n"
# "4. RESTful风格与可读性 (名词表示资源,HTTP方法表示动作,易理解性)。"
# )
# severity = TestSeverity.MEDIUM # 综合性检查,可能包含不同严重级别的问题
# tags = ["normative-spec", "url", "restful", "llm", "readability", "naming-convention", "structure", "versioning", "static-check"]
# execution_order = 60 # 更新执行顺序
# # 此测试用例可以覆盖所有路径,但其有效性依赖LLM
# # applicable_methods = None
# # applicable_paths_regex = None
# # 这个标志可以用来在测试用例级别控制是否实际调用LLM,即使全局LLM服务可用
# # 如果您希望总是尝试(只要LLMService能初始化),可以不设置这个,或者在逻辑中检查 self.llm_service 是否存在
# # use_llm_for_path_analysis: bool = True
# def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[LLMService] = None):
# super().__init__(endpoint_spec, global_api_spec, json_schema_validator)
# self.llm_service = llm_service # 存储注入的 LLMService 实例
# if not self.llm_service:
# self.logger.warning(f"LLMService 未注入或初始化失败,测试用例 {self.id} 将无法执行LLM路径分析。")
# def _get_llm_service_from_orchestrator(self) -> Optional[Any]:
# # 在实际框架中,测试用例可能无法直接访问编排器来获取LLM服务。
# # 这种依赖注入通常在测试用例实例化时或方法调用时处理。
# # 此处为一个占位符,理想情况下APITestOrchestrator会将llm_service实例传给需要它的测试用例,
# # 或测试用例通过某种服务定位器获取。
# # 暂时我们假设,如果全局配置了LLM,它就能用。
# # 真实的实现需要APITestOrchestrator在执行此测试用例前,将llm_service实例注入。
# # 为了能运行,我们先返回None,并在下面逻辑中处理。
# # 或者,修改 Orchestrator 将其注入到 self.global_api_spec 或 self.endpoint_spec (不推荐)
# # 最好的方式是在 __init__ 中接收一个 llm_service: Optional[LLMService] 参数。
# # 但这需要修改 BaseAPITestCase 和 APITestOrchestrator 的 __init__ 和调用逻辑。
# # 临时的解决方法:依赖 APITestOrchestrator 初始化时是否成功创建了 LLMService。
# # 这仍然是一个间接的检查。一个更直接的方式是在Orchestrator执行此测试用例时传入。
# if hasattr(self, '_orchestrator_llm_service_instance') and self._orchestrator_llm_service_instance:
# return self._orchestrator_llm_service_instance
# # 如果没有明确注入,我们只能依赖全局LLMService是否被加载
# if LLMService is not None:
# # 这里不能直接实例化一个新的LLMService,因为它需要API Key等配置,这些配置在Orchestrator那里。
# # 这个测试用例需要依赖Orchestrator来提供一个已经配置好的LLMService实例。
# # 此处返回一个指示:如果LLM功能应该被使用,则需要Orchestrator提供服务。
# return "NEEDS_INJECTION"
# return None
# def _extract_path_param_names(self, path_template: str) -> List[str]:
# """从路径模板中提取路径参数名称。例如 /users/{user_id}/items/{item_id} -> ['user_id', 'item_id']"""
# return re.findall(r'\{([^}]+)\}', path_template)
# def validate_request_url(self, url: str, request_context: APIRequestContext) -> List[ValidationResult]:
# results: List[ValidationResult] = []
# path_template = self.endpoint_spec.get('path', '')
# http_method = request_context.method.upper()
# operation_id = self.endpoint_spec.get('operationId', self.endpoint_spec.get('title', '')) # 获取operationId或title
# if not self.llm_service:
# results.append(ValidationResult(
# passed=True, # 标记为通过以避免阻塞,但消息表明跳过
# message=f"路径 '{path_template}' 的LLM综合URL检查已跳过:LLM服务不可用。",
# details={"path_template": path_template, "http_method": http_method, "reason": "LLM Service not available or not injected."}
# ))
# self.logger.warning(f"LLM综合URL检查已跳过对路径 '{path_template}' 的检查:LLM服务不可用。")
# return results
# path_param_names = self._extract_path_param_names(path_template)
# path_params_str = ", ".join(path_param_names) if path_param_names else "无"
# # - 接口名称 (OperationId 或 Title): {operation_id if operation_id else '请你自己从路径模板中提取'}
# # 构建给LLM的Prompt,要求JSON输出
# prompt_instruction = f"""
# 请扮演一位资深的API设计评审员。我将提供一个API端点的路径模板、HTTP方法以及可能的接口名称。
# 请根据以下石油行业API设计规范评估此API端点,并以严格的JSON格式返回您的评估结果。
# JSON对象应包含一个名为 "assessments" 的键,其值为一个对象列表,每个对象代表对一个标准的评估,包含 "standard_name" (字符串), "is_compliant" (布尔值), 和 "reason" (字符串) 三个键。
# API端点信息:
# - HTTP方法: {http_method}
# - 路径模板: {path_template}
# - 路径中提取的参数名: [{path_params_str}]
# 评估标准:
# 1. **接口名称规范 (接口名称需要你从路径模板中提取,一般是路径中除了参数名以外的最后的一个单词)**:
# - 规则: 采用'动词+名词'结构,明确业务语义 (例如: GetWellLog, SubmitSeismicJob)。
# - standard_name: "interface_naming_convention"
# 2. **HTTP方法使用规范**:
# - 规则: 遵循RESTful规范:GET用于数据检索, POST用于创建资源, PUT用于更新资源, DELETE用于删除资源。
# - standard_name: "http_method_usage"
# 3. **URL路径结构规范**:
# - 规则: 格式为 `<前缀>/<专业领域>/v<版本号>/<资源类型>` (例如: /logging/v1.2/wells, /seismicprospecting/v1.0/datasets)。
# - 前缀: 示例: /api/dms
# - 专业领域: 专业领域示例: seismicprospecting, welllogging, reservoirevaluation
# - 版本号: 语义化版本,例如 v1, v1.0, v2.1.3。
# - 资源类型: 通常为名词复数。
# - standard_name: "url_path_structure"
# 4. **URL路径参数命名规范**:
# - 规则: 路径参数(如果存在)必须使用全小写字母(可以是一个单词)或小写字母加下划线命名(这是多个单词的情况),并能反映资源的唯一标识 (例如: {{well_id}},还有{{version}},{{schema}}也是合规的,比一定非要{{version_id}})。
# - standard_name: "url_path_parameter_naming"
# 5. **资源命名规范 (在路径中)**:
# - 规则: 资源集合应使用名词的复数形式表示 (例如 `/wells`, `/logs`);应优先使用石油行业的标准术语 (例如用 `trajectory` 而非 `path` 来表示井轨迹)。
# - standard_name: "resource_naming_in_path"
# 请确保您的输出是一个可以被 `json.loads()` 直接解析的JSON对象。
# 例如:
# {{
# "assessments": [
# {{
# "standard_name": "interface_naming_convention",
# "is_compliant": true,
# "reason": "接口名称 'GetWellboreTrajectory' 符合动词+名词结构。"
# }},
# {{
# "standard_name": "http_method_usage",
# "is_compliant": true,
# "reason": "GET方法用于检索资源,符合规范。"
# }}
# // ... 其他标准的评估 ...
# ]
# }}
# """
# # 6. **路径可读性与整体RESTful风格**:
# # - 规则: 路径整体是否具有良好的可读性、易于理解其功能,并且符合RESTful设计原则?(综合评估,可参考前面几点)
# # - standard_name: "general_readability_and_restfulness"
# messages = [
# {"role": "system", "content": "你是一位API设计评审专家,专注于评估API的URL规范性和RESTful风格。你的输出必须是严格的JSON格式。"},
# {"role": "user", "content": prompt_instruction}
# ]
# self.logger.info(f"向LLM发送请求,评估路径: {path_template} ({http_method})")
# # 假设 _execute_chat_completion_request 支持 response_format={"type": "json_object"} (如果LLM API支持)
# # 否则,我们需要解析文本输出。为简化,这里假设LLM会遵循JSON格式指令。
# llm_response_str = self.llm_service._execute_chat_completion_request(
# messages=messages,
# max_tokens=1024, # 根据评估结果的复杂度调整
# temperature=0.2 # 低温以获得更确定的、结构化的输出
# )
# if not llm_response_str:
# results.append(ValidationResult(
# passed=False, # 执行失败
# message=f"未能从LLM获取对路径 '{path_template}' 的评估。",
# details={"path_template": path_template, "http_method": http_method, "reason": "LLM did not return a response."}
# ))
# self.logger.error(f"LLM对路径 '{path_template}' 的评估请求未返回任何内容。")
# return results
# self.logger.debug(f"LLM对路径 '{path_template}' 的原始响应: {llm_response_str}")
# try:
# # 尝试清理并解析LLM响应
# # 有时LLM可能在JSON前后添加 "```json" 和 "```"
# cleaned_response_str = llm_response_str.strip()
# if cleaned_response_str.startswith("```json"):
# cleaned_response_str = cleaned_response_str[7:]
# if cleaned_response_str.endswith("```"):
# cleaned_response_str = cleaned_response_str[:-3]
# llm_assessment_data = json.loads(cleaned_response_str)
# if "assessments" not in llm_assessment_data or not isinstance(llm_assessment_data["assessments"], list):
# raise ValueError("LLM响应JSON中缺少 'assessments' 列表或格式不正确。")
# found_assessments = False
# for assessment in llm_assessment_data["assessments"]:
# standard_name = assessment.get("standard_name", "未知标准")
# is_compliant = assessment.get("is_compliant", False)
# reason = assessment.get("reason", "LLM未提供原因。")
# found_assessments = True
# results.append(ValidationResult(
# passed=is_compliant,
# message=f"LLM评估 - {standard_name}: {reason}",
# details={
# "standard_name": standard_name,
# "is_compliant_by_llm": is_compliant,
# "llm_reason": reason,
# "path_template": path_template,
# "http_method": http_method
# }
# ))
# log_level = self.logger.info if is_compliant else self.logger.warning
# log_level(f"LLM评估 - 标准 '{standard_name}' for '{path_template}': {'符合' if is_compliant else '不符合'}。原因: {reason}")
# if not found_assessments:
# results.append(ValidationResult(
# passed=False,
# message=f"LLM返回的评估结果中不包含任何有效的评估项。",
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str}
# ))
# except json.JSONDecodeError as e_json:
# results.append(ValidationResult(
# passed=False, # 执行失败
# message=f"无法将LLM对路径 '{path_template}' 的评估响应解析为JSON: {e_json}",
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str, "error": str(e_json)}
# ))
# self.logger.error(f"LLM对路径 '{path_template}' 的响应JSON解析失败: {e_json}. Raw response: {llm_response_str}")
# except ValueError as e_val: # 自定义错误,如缺少 'assessments'
# results.append(ValidationResult(
# passed=False, # 执行失败
# message=f"LLM对路径 '{path_template}' 的评估响应JSON结构不符合预期: {e_val}",
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str, "error": str(e_val)}
# ))
# self.logger.error(f"LLM对路径 '{path_template}' 的响应JSON结构错误: {e_val}. Raw response: {llm_response_str}")
# except Exception as e_generic:
# results.append(ValidationResult(
# passed=False, # 执行失败
# message=f"处理LLM对路径 '{path_template}' 的评估响应时发生未知错误: {e_generic}",
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str, "error": str(e_generic)}
# ))
# self.logger.error(f"处理LLM对路径 '{path_template}' 的响应时发生未知错误: {e_generic}", exc_info=True)
# return results
@@ -0,0 +1 @@
# This file marks the security directory as a Python package.
@@ -0,0 +1,84 @@
from typing import Dict, Any, Optional, List
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import urllib.parse
class HTTPSMandatoryCase(BaseAPITestCase):
id = "TC-SECURITY-001"
name = "HTTPS 协议强制性检查"
description = "验证API端点是否通过HTTPS提供服务,以及HTTP请求是否被拒绝或重定向到HTTPS。"
severity = TestSeverity.HIGH
tags = ["security", "https", "transport-security"]
execution_order = 120
# 此测试会修改URL为HTTP,应适用于大多数端点。
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
def modify_request_url(self, current_url: str) -> str:
parsed_url = urllib.parse.urlparse(current_url)
if parsed_url.scheme.lower() == "https":
# 将 https 替换为 http
modified_url = parsed_url._replace(scheme="http").geturl()
self.logger.info(f"为进行HTTPS检查修改URL:原始 '{current_url}', 修改为 '{modified_url}'")
return modified_url
else:
self.logger.warning(f"原始URL '{current_url}' 不是HTTPS。跳过此测试用例的URL修改。")
# 如果原始URL不是HTTPS,此测试可能无效,
# 或者暗示基础URL本身可能未正确配置以进行HTTPS测试。
return current_url # 如果不是HTTPS则返回原始URL
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
results = []
status_code = response_context.status_code
# request_context.url 是调用 modify_request_url 之后,APICaller实际发送的URL
request_url_scheme = urllib.parse.urlparse(request_context.url).scheme
# 检查URL是否确实被此测试的钩子修改为了HTTP
if request_url_scheme.lower() != "http":
results.append(self.passed(
message=f"测试已跳过,因为发送的URL已经是 {request_url_scheme.upper()}(可能是由于原始基础URL非HTTPS或测试设置问题)。"
))
self.logger.info("HTTPS强制性检查已跳过,因为有效URL不是HTTP。")
return results
# 如果请求是通过HTTP发出的,我们期望几种结果:
# 1. 拒绝(例如400、403,或连接被拒绝 - 尽管APICaller可能在此之前处理连接拒绝)
# 2. 重定向到HTTPS(例如301、302、307、308,并带有指向HTTPS的Location头)
if status_code in [301, 302, 307, 308]: # 重定向状态码
location_header = response_context.headers.get("Location")
if location_header and urllib.parse.urlparse(location_header).scheme.lower() == "httpss":
results.append(self.passed(
message=f"{request_context.url} 的HTTP请求被正确重定向到HTTPS ({location_header}),状态码 {status_code}"
))
self.logger.info(f"HTTP被正确重定向到HTTPS: {location_header}")
else:
results.append(self.failed(
message=f"{request_context.url} 的HTTP请求被重定向(状态码 {status_code}),但Location头 '{location_header}' 未指向HTTPS URL。",
details={"status_code": status_code, "location_header": location_header}
))
self.logger.warning(f"HTTP被重定向,状态码 {status_code},但Location '{location_header}' 不是HTTPS。")
elif status_code // 100 == 4: # 客户端错误(例如400错误请求,403禁止访问)
results.append(self.passed(
message=f"{request_context.url} 的HTTP请求被客户端错误(状态码 {status_code})拒绝,表明不允许HTTP访问。"
))
self.logger.info(f"HTTP请求被客户端错误 {status_code} 拒绝。")
elif status_code // 100 == 2: # 通过HTTP成功返回2xx响应
results.append(self.failed(
message=f"API通过HTTP ({request_context.url}) 响应了成功的状态码 {status_code},这违反了HTTPS强制策略。",
details={"status_code": status_code}
))
self.logger.error(f"安全漏洞:API允许通过HTTP成功响应 ({status_code})。")
else:
# 其他状态码(例如5xx)可能表示与HTTPS强制执行无关的服务器错误,
# 或者连接在更底层被拒绝(APICaller可能会抛出异常)。
results.append(ValidationResult(
passed=False, # 或True,取决于严格程度 - 5xx不是通过,但不一定是HTTPS失败
message=f"{request_context.url} 的HTTP请求返回了意外的状态码 {status_code}。需要手动调查。",
details={"status_code": status_code, "response_text_sample": (response_context.text_content or "")[:200]}
))
self.logger.warning(f"HTTP请求返回意外状态码 {status_code}。潜在问题或服务器错误。")
return results