report
This commit is contained in:
@@ -0,0 +1 @@
|
||||
# This file marks the compliance_catalog directory as a Python package.
|
||||
@@ -0,0 +1 @@
|
||||
# This file marks the core_functionality directory as a Python package.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
+81
@@ -0,0 +1,81 @@
|
||||
from typing import Dict, Any, List, Optional
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext
|
||||
from ddms_compliance_suite.utils.response_utils import extract_data_for_validation
|
||||
from ddms_compliance_suite.utils.schema_provider import SchemaProvider
|
||||
import logging
|
||||
|
||||
class FlexibleSchemaValidationCase(BaseAPITestCase):
|
||||
"""
|
||||
一个灵活的Schema验证测试用例,能够处理非标准的响应结构和动态的Schema来源。
|
||||
"""
|
||||
id = "TC-CORE-FUNC-002"
|
||||
name = "灵活的返回体JSON Schema验证"
|
||||
description = (
|
||||
"验证API响应体是否符合预期的JSON Schema。此用例能够智能处理被包装的响应(如{code, data}),"
|
||||
"并支持从列表响应中验证每个元素。它依赖于SchemaProvider获取schema,并设计为处理需要动态获取schema的场景。"
|
||||
)
|
||||
severity = TestSeverity.CRITICAL
|
||||
tags = ["core-functionality", "schema-validation", "flexible"]
|
||||
execution_order = 110 # 略高于标准Schema验证,以便在适用时优先执行
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service)
|
||||
# We need to initialize the schema_provider here, as it's no longer injected.
|
||||
self.schema_provider = SchemaProvider(global_api_spec) if global_api_spec else None
|
||||
self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
|
||||
|
||||
|
||||
def execute(self, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""
|
||||
执行灵活的schema验证。
|
||||
"""
|
||||
results = []
|
||||
response_context = self.api_caller.call_api(request_context)
|
||||
|
||||
if not response_context:
|
||||
return [self.failed("API调用失败,无法获取响应进行验证。")]
|
||||
|
||||
# 1. 使用 SchemaProvider 获取 Schema
|
||||
if not self.schema_provider:
|
||||
return [self.failed("SchemaProvider 未被初始化,无法执行此测试用例。")]
|
||||
|
||||
expected_schema = self.schema_provider.get_schema(self.endpoint_spec, response_context.status_code)
|
||||
|
||||
if not expected_schema:
|
||||
# 如果是成功响应但找不到schema,这可能是一个问题
|
||||
if 200 <= response_context.status_code < 300:
|
||||
return [self.failed(f"成功响应(状态码 {response_context.status_code}),但无法为其找到或生成JSON Schema。")]
|
||||
else:
|
||||
return [self.passed(f"非成功响应(状态码 {response_context.status_code})且未定义Schema,跳过验证。")]
|
||||
|
||||
# 2. 使用 response_utils 提取待验证的数据列表
|
||||
if not response_context.json_content:
|
||||
return [self.failed(f"响应内容不是有效的JSON格式,无法进行Schema验证。响应文本: {response_context.text_content[:200]}...")]
|
||||
|
||||
data_to_validate_list = extract_data_for_validation(response_context.json_content)
|
||||
|
||||
if not data_to_validate_list:
|
||||
# extract_data_for_validation 在找不到数据或遇到空列表时返回空列表
|
||||
return [self.passed("未从响应中提取到需要验证的数据项(可能为空列表),跳过验证。")]
|
||||
|
||||
# 3. 遍历列表,对每个数据项进行验证
|
||||
all_items_passed = True
|
||||
for i, item in enumerate(data_to_validate_list):
|
||||
item_context_prefix = f"响应列表中的第 {i+1} 个元素"
|
||||
validation_results = self.validate_data_against_schema(
|
||||
data_to_validate=item,
|
||||
schema_definition=expected_schema,
|
||||
context_message_prefix=item_context_prefix
|
||||
)
|
||||
|
||||
for res in validation_results:
|
||||
if not res.passed:
|
||||
all_items_passed = False
|
||||
# 为错误信息添加更多上下文
|
||||
res.message = f"{item_context_prefix} {res.message}"
|
||||
results.append(res)
|
||||
|
||||
if all_items_passed:
|
||||
results.append(self.passed(f"成功验证了响应中的 {len(data_to_validate_list)} 个数据项,均符合Schema。"))
|
||||
|
||||
return results
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
from typing import Dict, Any, Optional, List
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import json
|
||||
from ddms_compliance_suite.utils.schema_provider import SchemaProvider
|
||||
|
||||
class ResponseSchemaValidationCase(BaseAPITestCase):
|
||||
id = "TC-CORE-FUNC-001"
|
||||
name = "返回体JSON Schema验证"
|
||||
description = "验证API响应体是否符合API规范中定义的JSON Schema。"
|
||||
severity = TestSeverity.CRITICAL
|
||||
tags = ["core-functionality", "schema-validation", "output-format"]
|
||||
execution_order = 100 # Default, can be adjusted
|
||||
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.schema_provider = SchemaProvider(global_api_spec) if global_api_spec else None
|
||||
self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
results = []
|
||||
method = request_context.method.upper()
|
||||
status_code = response_context.status_code
|
||||
|
||||
if not self.schema_provider:
|
||||
return [self.failed("SchemaProvider 未被初始化,无法执行此测试用例。")]
|
||||
|
||||
expected_schema = self.schema_provider.get_schema(self.endpoint_spec, status_code)
|
||||
|
||||
if expected_schema and response_context.json_content is not None:
|
||||
self.logger.info(f"将根据从API规范中获取的schema验证响应体。")
|
||||
schema_validation_results = self.validate_data_against_schema(
|
||||
data_to_validate=response_context.json_content,
|
||||
schema_definition=expected_schema,
|
||||
context_message_prefix=f"针对 {method} {request_context.url} (状态码 {status_code}) 的响应体"
|
||||
)
|
||||
results.extend(schema_validation_results)
|
||||
elif response_context.json_content is None and method not in ["DELETE", "HEAD", "OPTIONS"] and status_code in [200, 201, 202]:
|
||||
if expected_schema:
|
||||
results.append(self.failed(
|
||||
message=f"根据schema期望一个JSON响应体,但未收到可解析的JSON内容。",
|
||||
details={"status_code": status_code, "response_text_sample": (response_context.text_content or "")[:200]}
|
||||
))
|
||||
self.logger.warning(f"期望 {method} {request_context.url} 返回JSON响应体,但未收到或非JSON格式。")
|
||||
elif not expected_schema and response_context.json_content is not None and status_code // 100 == 2:
|
||||
self.logger.info(f"响应包含JSON体,但在API规范中未找到针对状态码 {status_code} 的JSON schema。跳过schema验证。")
|
||||
elif not expected_schema and response_context.json_content is None:
|
||||
self.logger.info(f"状态码 {status_code} 的响应无JSON体也无定义的schema。跳过schema验证。")
|
||||
|
||||
if not results:
|
||||
results.append(self.passed("标准Schema验证步骤完成(未发现问题,或schema不适用/未为此响应定义)。"))
|
||||
|
||||
return results
|
||||
@@ -0,0 +1 @@
|
||||
# This file marks the error_handling directory as a Python package.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
+196
@@ -0,0 +1,196 @@
|
||||
from typing import Dict, Any, Optional, List, Union, Tuple
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import logging
|
||||
from ddms_compliance_suite.utils import schema_utils
|
||||
import random
|
||||
import string
|
||||
|
||||
class InvalidEnumValueCase(BaseAPITestCase):
|
||||
id = "TC-ERROR-4006"
|
||||
name = "非法枚举值检查"
|
||||
description = "测试当发送的参数值不在指定的枚举范围内时,API是否按预期返回code=4006的错误。"
|
||||
severity = TestSeverity.MEDIUM
|
||||
tags = ["error-handling", "appendix-b", "4006", "invalid-enum"]
|
||||
execution_order = 204 # 在数值越界之后执行
|
||||
|
||||
@staticmethod
|
||||
def _endpoint_has_enum_field(endpoint_spec: Dict[str, Any], logger: logging.Logger) -> bool:
|
||||
"""
|
||||
静态辅助方法,检查端点规范中是否有任何字段(body或parameters)包含枚举定义。
|
||||
"""
|
||||
# 1. 检查请求体 (Body)
|
||||
if endpoint_spec.get("requestBody"):
|
||||
# 注意:这里需要一个可以解析$ref的上下文,但applies_to是静态的。
|
||||
# 这是一个简化检查,它只检查顶级schema。
|
||||
# 更完整的检查需要一个能够解析$ref的schema工具。
|
||||
# 为了简单起见,我们假设测试用例的 _get_resolved_request_body_schema 可以在实例化后使用。
|
||||
# 这里的检查可能不完整,但可以覆盖大部分情况。
|
||||
if schema_utils.util_find_enum_field_recursive(endpoint_spec["requestBody"].get("content", {}).get("application/json", {}).get("schema", {}), [], logger):
|
||||
return True
|
||||
|
||||
# 2. 检查查询参数, 路径参数, 请求头
|
||||
if 'parameters' in endpoint_spec:
|
||||
for param in endpoint_spec['parameters']:
|
||||
param_desc = param.get('description', '')
|
||||
if schema_utils.util_extract_enum_from_description(param_desc, logger):
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
|
||||
"""
|
||||
此测试用例仅适用于那些在请求体或参数中定义了枚举(enum)的端点。
|
||||
"""
|
||||
# 创建一个临时logger用于静态方法
|
||||
temp_logger = logging.getLogger(f"applies_to.{cls.id}")
|
||||
return cls._endpoint_has_enum_field(endpoint_spec, temp_logger)
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.logger.setLevel(logging.INFO)
|
||||
|
||||
self.target_field_location: Optional[str] = None
|
||||
self.target_field_path: Optional[List[Union[str, int]]] = None
|
||||
self.target_field_name: Optional[str] = None
|
||||
self.valid_enums: Optional[List[str]] = None
|
||||
self.invalid_enum_value: Optional[str] = None
|
||||
|
||||
self._initialize_test_case()
|
||||
|
||||
def _find_first_enum_field(self) -> Optional[Tuple[str, List[Union[str, int]], str, List[str], str]]:
|
||||
"""
|
||||
按优先级(body -> query -> path -> header)查找第一个具有枚举值限制的字段。
|
||||
"""
|
||||
self.logger.info(f"[{self.id}] Searching for a field with enum constraints...")
|
||||
|
||||
# 1. 检查请求体 (Body)
|
||||
body_schema = self._get_resolved_request_body_schema()
|
||||
if body_schema:
|
||||
self.logger.debug(f"[{self.id}] Checking for enum field in request body.")
|
||||
result = schema_utils.util_find_enum_field_recursive(body_schema, [], self.logger)
|
||||
if result:
|
||||
path, valid_enums, desc = result
|
||||
name = path[-1] if path else "body_root"
|
||||
self.logger.info(f"[{self.id}] Found enum field in BODY at path: {'.'.join(map(str, path))}")
|
||||
return "body", path, name, valid_enums, desc
|
||||
|
||||
# 2. 检查查询参数, 路径参数, 请求头
|
||||
for location in ["query", "path", "header"]:
|
||||
if 'parameters' in self.endpoint_spec:
|
||||
self.logger.debug(f"[{self.id}] Checking for enum field in {location} parameters.")
|
||||
for param in self.endpoint_spec['parameters']:
|
||||
if param.get('in') == location:
|
||||
param_desc = param.get('description', '')
|
||||
param_name = param.get('name')
|
||||
|
||||
valid_enums = schema_utils.util_extract_enum_from_description(param_desc, self.logger)
|
||||
|
||||
if valid_enums:
|
||||
self.logger.info(f"[{self.id}] Found enum field in {location.upper()}: '{param_name}'")
|
||||
return location, [param_name], param_name, valid_enums, param_desc
|
||||
|
||||
self.logger.info(f"[{self.id}] No field with enum constraints found in any location.")
|
||||
return None
|
||||
|
||||
def _initialize_test_case(self):
|
||||
"""初始化测试用例。"""
|
||||
found_field = self._find_first_enum_field()
|
||||
|
||||
if found_field:
|
||||
location, path, name, valid_enums, description = found_field
|
||||
|
||||
self.target_field_location = location
|
||||
self.target_field_path = path
|
||||
self.target_field_name = name
|
||||
self.valid_enums = valid_enums
|
||||
|
||||
self.logger.info(
|
||||
f"[{self.id}] Target found in '{location}' at path '{'.'.join(map(str, path))}'. "
|
||||
f"Valid enums: {valid_enums}, Desc: '{description}'"
|
||||
)
|
||||
|
||||
self._set_invalid_enum_value()
|
||||
else:
|
||||
self.logger.info(f"[{self.id}] No suitable enum field found for this endpoint.")
|
||||
|
||||
def _set_invalid_enum_value(self):
|
||||
"""生成一个非法的枚举值。"""
|
||||
random_suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=6))
|
||||
self.invalid_enum_value = f"invalid_enum_{random_suffix}"
|
||||
self.logger.info(f"[{self.id}] Setting invalid enum value to '{self.invalid_enum_value}' (valid are {self.valid_enums}).")
|
||||
|
||||
def _modify_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""通用函数,用于修改参数字典中的值。"""
|
||||
if self.invalid_enum_value is None or not self.target_field_path:
|
||||
return params
|
||||
|
||||
modified_params, success = schema_utils.util_set_value_at_path(
|
||||
data_container=params,
|
||||
path=self.target_field_path,
|
||||
new_value=self.invalid_enum_value
|
||||
)
|
||||
|
||||
if success:
|
||||
self.logger.debug(f"[{self.id}] Successfully injected invalid enum value into parameters.")
|
||||
return modified_params
|
||||
|
||||
self.logger.error(f"[{self.id}] Failed to set invalid enum value in parameters.")
|
||||
return params
|
||||
|
||||
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if self.target_field_location == "query":
|
||||
return self._modify_params(current_query_params)
|
||||
return current_query_params
|
||||
|
||||
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if self.target_field_location == "path":
|
||||
return self._modify_params(current_path_params)
|
||||
return current_path_params
|
||||
|
||||
def generate_headers(self, current_headers: Dict[str, str]) -> Dict[str, str]:
|
||||
if self.target_field_location == "header":
|
||||
return self._modify_params(current_headers)
|
||||
return current_headers
|
||||
|
||||
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
|
||||
if self.target_field_location == "body":
|
||||
return self._modify_params(current_body)
|
||||
return current_body
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""验证API是否正确返回code=4006错误(状态码为200)。"""
|
||||
results = []
|
||||
|
||||
if self.invalid_enum_value is None:
|
||||
return [self.passed("跳过测试:未找到具有明确枚举值限制的字段。")]
|
||||
|
||||
expected_http_status_code = 200
|
||||
expected_business_error_code = -1
|
||||
|
||||
context_msg_prefix = (
|
||||
f"当 {self.target_field_location} 字段 '{self.target_field_name}' "
|
||||
f"值为 '{self.invalid_enum_value}' (合法值为: {self.valid_enums}) 时, "
|
||||
)
|
||||
|
||||
# 检查状态码
|
||||
if response_context.status_code != expected_http_status_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}。",
|
||||
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
|
||||
)]
|
||||
|
||||
# 检查业务错误码
|
||||
json_content = response_context.json_content
|
||||
if not isinstance(json_content, dict):
|
||||
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
|
||||
|
||||
actual_business_code = json_content.get("code")
|
||||
if actual_business_code != expected_business_error_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}。",
|
||||
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
|
||||
)]
|
||||
|
||||
return [self.passed(
|
||||
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}。"
|
||||
)]
|
||||
Vendored
+98
@@ -0,0 +1,98 @@
|
||||
from typing import Dict, Any, Optional, List, Union
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import logging
|
||||
from ddms_compliance_suite.utils import schema_utils # Keep this import for util_remove_value_at_path
|
||||
|
||||
class MissingRequiredFieldBodyCase(BaseAPITestCase):
|
||||
id = "TC-ERROR-4003-BODY"
|
||||
name = "缺失必填请求体字段检查"
|
||||
description = "测试当请求体中缺少API规范定义的必填字段时,API是否按预期返回code=4003的错误。"
|
||||
severity = TestSeverity.HIGH
|
||||
tags = ["error-handling", "appendix-b", "4003", "required-fields", "request-body"]
|
||||
execution_order = 210
|
||||
|
||||
@classmethod
|
||||
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
|
||||
"""
|
||||
此测试用例仅适用于那些在API规范中明确定义了请求体(requestBody)的端点。
|
||||
"""
|
||||
# 如果 'requestBody' 存在且其内容不为空,则认为此测试用例适用。
|
||||
return bool(endpoint_spec.get("requestBody"))
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service)
|
||||
self.logger = logging.getLogger(f"testcase.{self.id}") # Already set in super, but can be re-set if specific sub-logger is needed. Better to rely on super's logger.
|
||||
self.original_value_at_path: Any = None
|
||||
|
||||
# Use new helper methods from BaseAPITestCase
|
||||
body_schema = self._get_resolved_request_body_schema()
|
||||
if body_schema:
|
||||
self.removed_field_path = self._find_removable_field_path(
|
||||
schema_to_search=body_schema,
|
||||
schema_name_for_log="request body"
|
||||
)
|
||||
if not self.removed_field_path:
|
||||
self.logger.info('在请求体 schema 中未找到可用于测试 "必填字段缺失" 的字段(通过基类方法)。')
|
||||
else:
|
||||
self.logger.info('此端点规范中未定义或找到请求体 schema(通过基类方法)。')
|
||||
self.removed_field_path = None
|
||||
|
||||
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
self.logger.debug(f"{self.id} is focused on request body, generate_query_params will not modify query parameters.")
|
||||
return current_query_params
|
||||
|
||||
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
|
||||
if not self.removed_field_path:
|
||||
self.logger.debug("No field path identified for removal in request body. Returning original body.")
|
||||
return current_body
|
||||
|
||||
modified_body, self.original_value_at_path, success = schema_utils.util_remove_value_at_path(
|
||||
data_container=current_body,
|
||||
path=self.removed_field_path
|
||||
)
|
||||
|
||||
if success:
|
||||
self.logger.info(f"为进行必填字段缺失测试,已通过工具方法从请求体中移除字段路径 '{'.'.join(map(str, self.removed_field_path))}'。")
|
||||
return modified_body
|
||||
else:
|
||||
self.logger.error(f"使用工具方法移除请求体字段路径 '{'.'.join(map(str, self.removed_field_path))}' 失败。将返回原始请求体。")
|
||||
# Restore original_value_at_path to None since removal failed
|
||||
self.original_value_at_path = None
|
||||
return current_body
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""验证API是否正确返回code=4003错误(状态码为200)。"""
|
||||
results = []
|
||||
if not self.removed_field_path:
|
||||
results.append(self.passed("跳过测试:在API规范中未找到合适的必填请求体字段用于移除测试。"))
|
||||
self.logger.info("由于未识别到可移除的必填请求体字段,跳过此测试用例的验证。")
|
||||
return results
|
||||
|
||||
expected_http_status_code = 200
|
||||
expected_business_error_code = -1
|
||||
|
||||
removed_field_str = '.'.join(map(str, self.removed_field_path))
|
||||
context_msg_prefix = f"当移除必填请求体字段 '{removed_field_str}' 时, "
|
||||
|
||||
# 检查状态码
|
||||
if response_context.status_code != expected_http_status_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}。",
|
||||
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
|
||||
)]
|
||||
|
||||
# 检查业务错误码
|
||||
json_content = response_context.json_content
|
||||
if not isinstance(json_content, dict):
|
||||
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
|
||||
|
||||
actual_business_code = json_content.get("code")
|
||||
if actual_business_code != expected_business_error_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}。",
|
||||
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
|
||||
)]
|
||||
|
||||
return [self.passed(
|
||||
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}。"
|
||||
)]
|
||||
Vendored
+92
@@ -0,0 +1,92 @@
|
||||
from typing import Dict, Any, Optional, List
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import copy
|
||||
|
||||
class MissingRequiredFieldQueryCase(BaseAPITestCase):
|
||||
id = "TC-ERROR-4003-QUERY"
|
||||
name = "缺失必填查询参数检查"
|
||||
description = "测试当请求中缺少API规范定义的必填查询参数时,API是否按预期返回code=4003的错误。"
|
||||
severity = TestSeverity.HIGH
|
||||
tags = ["error-handling", "appendix-b", "4003", "required-fields", "query-parameters"]
|
||||
execution_order = 211 # After body, before original combined one might have been
|
||||
|
||||
@classmethod
|
||||
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
|
||||
"""
|
||||
此测试用例仅适用于那些在API规范中定义了查询参数(parameters with 'in' == 'query')的端点。
|
||||
"""
|
||||
# 如果 'parameters' 字段不存在,则不适用
|
||||
if not endpoint_spec.get("parameters"):
|
||||
return False
|
||||
|
||||
# 遍历所有参数,如果发现任何一个参数的 "in" 字段是 "query",则此测试用例适用。
|
||||
return any(param.get("in") == "query" for param in endpoint_spec["parameters"])
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.target_param_name: Optional[str] = None
|
||||
# Location is always 'query' for this class
|
||||
self.target_param_location: str = "query"
|
||||
self.original_query_params: Optional[Dict[str, Any]] = None
|
||||
|
||||
# Call the simplified method to find the target parameter
|
||||
self._try_find_removable_query_param()
|
||||
self.logger.info(f"测试用例 {self.id} ({self.name}) 已针对端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。Target param to remove: {self.target_param_name}")
|
||||
|
||||
def _try_find_removable_query_param(self):
|
||||
"""Uses the base class helper to find a required query parameter."""
|
||||
self.target_param_name = self._find_required_parameter_name("query")
|
||||
# Logging about success/failure is handled by the base class method and the __init__ method.
|
||||
|
||||
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
|
||||
# This test case focuses on query parameters, so it does not modify the request body.
|
||||
self.logger.debug(f"{self.id} is focused on query parameters, generate_request_body will not modify the request body.")
|
||||
return current_body
|
||||
|
||||
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if self.target_param_name and isinstance(current_query_params, dict):
|
||||
if self.target_param_name in current_query_params:
|
||||
new_params = copy.deepcopy(current_query_params)
|
||||
original_value = new_params.pop(self.target_param_name) # 移除参数
|
||||
self.logger.info(f"为进行必填查询参数缺失测试,已从查询参数中移除 '{self.target_param_name}' (原值: '{original_value}')。")
|
||||
return new_params
|
||||
else:
|
||||
self.logger.warning(f"计划移除的查询参数 '{self.target_param_name}' 在当前查询参数中未找到。")
|
||||
return current_query_params
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""验证API是否正确返回code=4003错误(状态码为200)。"""
|
||||
results = []
|
||||
|
||||
if not self.target_param_name:
|
||||
results.append(self.passed("跳过测试:在API规范中未找到合适的必填查询参数用于移除测试。"))
|
||||
self.logger.info("由于未识别到可移除的必填查询参数,跳过此测试用例。")
|
||||
return results
|
||||
|
||||
expected_http_status_code = 200
|
||||
expected_business_error_code = -1
|
||||
|
||||
context_msg_prefix = f"当移除必填查询参数 '{self.target_param_name}' 时, "
|
||||
|
||||
# 检查状态码
|
||||
if response_context.status_code != expected_http_status_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}。",
|
||||
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
|
||||
)]
|
||||
|
||||
# 检查业务错误码
|
||||
json_content = response_context.json_content
|
||||
if not isinstance(json_content, dict):
|
||||
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
|
||||
|
||||
actual_business_code = json_content.get("code")
|
||||
if actual_business_code != expected_business_error_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}。",
|
||||
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
|
||||
)]
|
||||
|
||||
return [self.passed(
|
||||
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}。"
|
||||
)]
|
||||
+227
@@ -0,0 +1,227 @@
|
||||
from typing import Dict, Any, Optional, List, Union, Tuple
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import logging
|
||||
from ddms_compliance_suite.utils import schema_utils
|
||||
|
||||
class NumberOutOfRangeCase(BaseAPITestCase):
|
||||
id = "TC-ERROR-4002"
|
||||
name = "数值参数越界检查"
|
||||
description = "测试当发送的数值参数超出范围限制时,API是否按预期返回code=4002的错误。"
|
||||
severity = TestSeverity.MEDIUM
|
||||
tags = ["error-handling", "appendix-b", "4002", "out-of-range"]
|
||||
execution_order = 203 # 在类型不匹配测试之后执行
|
||||
|
||||
@staticmethod
|
||||
def _endpoint_has_ranged_field(endpoint_spec: Dict[str, Any], logger: logging.Logger) -> bool:
|
||||
"""
|
||||
静态辅助方法,检查端点规范中是否有任何字段(body或parameters)包含范围定义。
|
||||
"""
|
||||
# 1. 检查请求体 (Body)
|
||||
if endpoint_spec.get("requestBody"):
|
||||
if schema_utils.util_find_ranged_field_recursive(endpoint_spec["requestBody"].get("content", {}).get("application/json", {}).get("schema", {}), [], logger):
|
||||
return True
|
||||
|
||||
# 2. 检查查询参数, 路径参数, 请求头
|
||||
if 'parameters' in endpoint_spec:
|
||||
for param in endpoint_spec['parameters']:
|
||||
param_schema = param.get('schema', {})
|
||||
param_desc = param.get('description', '')
|
||||
|
||||
min_val, max_val = schema_utils.util_extract_range_from_description(param_desc, logger)
|
||||
|
||||
if min_val is None and max_val is None:
|
||||
min_val = param_schema.get('minimum')
|
||||
max_val = param_schema.get('maximum')
|
||||
|
||||
if min_val is not None or max_val is not None:
|
||||
return True
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
|
||||
"""
|
||||
此测试用例仅适用于那些在请求体或参数中定义了数值范围的端点。
|
||||
"""
|
||||
temp_logger = logging.getLogger(f"applies_to.{cls.id}")
|
||||
return cls._endpoint_has_ranged_field(endpoint_spec, temp_logger)
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
|
||||
self.target_field_location: Optional[str] = None
|
||||
self.target_field_path: Optional[List[Union[str, int]]] = None
|
||||
self.target_field_schema: Optional[Dict[str, Any]] = None
|
||||
self.target_field_name: Optional[str] = None
|
||||
self.target_field_type: Optional[str] = None
|
||||
self.min_value: Optional[float] = None
|
||||
self.max_value: Optional[float] = None
|
||||
self.out_of_range_value: Optional[Union[int, float]] = None
|
||||
|
||||
self._initialize_test_case()
|
||||
|
||||
def _find_first_ranged_numeric_field(self) -> Optional[Tuple[str, List[Union[str, int]], Dict[str, Any], str, Optional[str], Optional[float], Optional[float], str]]:
|
||||
"""
|
||||
按优先级(body -> query -> path -> header)查找第一个具有范围限制的数值字段。
|
||||
"""
|
||||
self.logger.info(f"[{self.id}] Searching for a ranged numeric field...")
|
||||
|
||||
# 1. 检查请求体 (Body)
|
||||
body_schema = self._get_resolved_request_body_schema()
|
||||
if body_schema:
|
||||
self.logger.debug(f"[{self.id}] Checking for ranged numeric field in request body.")
|
||||
result = schema_utils.util_find_ranged_field_recursive(body_schema, [], self.logger)
|
||||
if result:
|
||||
path, schema, type, min_val, max_val, desc = result
|
||||
name = path[-1] if path else "body_root"
|
||||
self.logger.info(f"[{self.id}] Found ranged numeric field in BODY at path: {'.'.join(map(str, path))}")
|
||||
return "body", path, schema, name, type, min_val, max_val, desc
|
||||
|
||||
# 2. 检查查询参数, 路径参数, 请求头
|
||||
for location in ["query", "path", "header"]:
|
||||
if 'parameters' in self.endpoint_spec:
|
||||
self.logger.debug(f"[{self.id}] Checking for ranged numeric field in {location} parameters.")
|
||||
for param in self.endpoint_spec['parameters']:
|
||||
if param.get('in') == location:
|
||||
param_schema = param.get('schema', {})
|
||||
param_desc = param.get('description', '')
|
||||
param_name = param.get('name')
|
||||
|
||||
min_val, max_val = schema_utils.util_extract_range_from_description(param_desc, self.logger)
|
||||
|
||||
if min_val is None and max_val is None:
|
||||
min_val = param_schema.get('minimum')
|
||||
max_val = param_schema.get('maximum')
|
||||
|
||||
if min_val is not None or max_val is not None:
|
||||
param_type = param_schema.get('type')
|
||||
self.logger.info(f"[{self.id}] Found ranged numeric field in {location.upper()}: '{param_name}'")
|
||||
return location, [param_name], param_schema, param_name, param_type, min_val, max_val, param_desc
|
||||
|
||||
self.logger.info(f"[{self.id}] No ranged numeric field found in any location.")
|
||||
return None
|
||||
|
||||
def _initialize_test_case(self):
|
||||
"""使用框架辅助方法初始化测试用例。"""
|
||||
found_field = self._find_first_ranged_numeric_field()
|
||||
|
||||
if found_field:
|
||||
location, path, schema, name, field_type, min_val, max_val, description = found_field
|
||||
|
||||
self.target_field_location = location
|
||||
self.target_field_path = path
|
||||
self.target_field_schema = schema
|
||||
self.target_field_name = name
|
||||
# 如果'type'未定义,我们根据是否存在小数来推断
|
||||
self.target_field_type = field_type if field_type else ('number' if isinstance(min_val, float) or isinstance(max_val, float) else 'integer')
|
||||
self.min_value = min_val
|
||||
self.max_value = max_val
|
||||
|
||||
self.logger.info(
|
||||
f"[{self.id}] Target found in '{location}' at path '{'.'.join(map(str, path))}'. "
|
||||
f"Type: {self.target_field_type}, Min: {min_val}, Max: {max_val}, Desc: '{description}'"
|
||||
)
|
||||
|
||||
self._set_out_of_range_value()
|
||||
else:
|
||||
self.logger.info(f"[{self.id}] No suitable ranged numeric field found for this endpoint.")
|
||||
|
||||
def _set_out_of_range_value(self):
|
||||
"""根据提取的范围信息设置越界值。"""
|
||||
# 优先选择超过最大值
|
||||
if self.max_value is not None:
|
||||
if self.target_field_type == 'integer':
|
||||
self.out_of_range_value = int(self.max_value) + 1
|
||||
else:
|
||||
self.out_of_range_value = self.max_value + 1.0
|
||||
self.logger.info(f"[{self.id}] Setting out-of-range value to {self.out_of_range_value} (max is {self.max_value}).")
|
||||
return
|
||||
|
||||
# 其次选择小于最小值
|
||||
if self.min_value is not None:
|
||||
if self.target_field_type == 'integer':
|
||||
self.out_of_range_value = int(self.min_value) - 1
|
||||
else:
|
||||
self.out_of_range_value = self.min_value - 1.0
|
||||
self.logger.info(f"[{self.id}] Setting out-of-range value to {self.out_of_range_value} (min is {self.min_value}).")
|
||||
|
||||
def _modify_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""通用函数,用于修改参数字典中的值。"""
|
||||
if self.out_of_range_value is None or not self.target_field_path:
|
||||
return params
|
||||
|
||||
modified_params, success = schema_utils.util_set_value_at_path(
|
||||
data_container=params,
|
||||
path=self.target_field_path,
|
||||
new_value=self.out_of_range_value
|
||||
)
|
||||
|
||||
if success:
|
||||
self.logger.debug(f"[{self.id}] Successfully injected out-of-range value into parameters.")
|
||||
return modified_params
|
||||
|
||||
self.logger.error(f"[{self.id}] Failed to set out-of-range value in parameters.")
|
||||
return params
|
||||
|
||||
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if self.target_field_location == "query":
|
||||
return self._modify_params(current_query_params)
|
||||
return current_query_params
|
||||
|
||||
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if self.target_field_location == "path":
|
||||
return self._modify_params(current_path_params)
|
||||
return current_path_params
|
||||
|
||||
def generate_headers(self, current_headers: Dict[str, str]) -> Dict[str, str]:
|
||||
if self.target_field_location == "header":
|
||||
# Header values must be strings
|
||||
original_value = self.out_of_range_value
|
||||
self.out_of_range_value = str(self.out_of_range_value)
|
||||
headers = self._modify_params(current_headers)
|
||||
self.out_of_range_value = original_value # revert for logging
|
||||
return headers
|
||||
return current_headers
|
||||
|
||||
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
|
||||
if self.target_field_location == "body":
|
||||
return self._modify_params(current_body)
|
||||
return current_body
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""验证API是否正确返回code=4002错误(状态码为200)。"""
|
||||
results = []
|
||||
|
||||
if self.out_of_range_value is None:
|
||||
return [self.passed("跳过测试:未找到具有明确范围限制的数值字段。")]
|
||||
|
||||
expected_http_status_code = 200
|
||||
expected_business_error_code = -1
|
||||
|
||||
context_msg_prefix = (
|
||||
f"当 {self.target_field_location} 字段 '{self.target_field_name}' "
|
||||
f"值为 {self.out_of_range_value} (超出范围: min={self.min_value}, max={self.max_value}) 时, "
|
||||
)
|
||||
|
||||
# 检查状态码
|
||||
if response_context.status_code != expected_http_status_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}。",
|
||||
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
|
||||
)]
|
||||
|
||||
# 检查业务错误码
|
||||
json_content = response_context.json_content
|
||||
if not isinstance(json_content, dict):
|
||||
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
|
||||
|
||||
actual_business_code = json_content.get("code")
|
||||
if actual_business_code != expected_business_error_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}。",
|
||||
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
|
||||
)]
|
||||
|
||||
return [self.passed(
|
||||
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}。"
|
||||
)]
|
||||
+125
@@ -0,0 +1,125 @@
|
||||
from typing import Dict, Any, Optional, List, Union
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import copy
|
||||
import logging
|
||||
from ddms_compliance_suite.utils import schema_utils
|
||||
|
||||
class TypeMismatchBodyCase(BaseAPITestCase):
|
||||
id = "TC-ERROR-4001-BODY"
|
||||
name = "请求体字段类型不匹配检查"
|
||||
description = "测试当发送的请求体中字段的数据类型与API规范定义不符时,API是否按预期返回code=4001的错误。"
|
||||
severity = TestSeverity.MEDIUM
|
||||
tags = ["error-handling", "appendix-b", "4001", "request-body"]
|
||||
execution_order = 202 # Slightly after query param one
|
||||
|
||||
@classmethod
|
||||
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
|
||||
"""
|
||||
此测试用例仅适用于那些在API规范中明确定义了请求体(requestBody)的端点。
|
||||
"""
|
||||
# 如果 'requestBody' 存在且其内容不为空,则认为此测试用例适用。
|
||||
return bool(endpoint_spec.get("requestBody"))
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
self.target_field_path: Optional[List[str]] = None
|
||||
self.original_field_type: Optional[str] = None
|
||||
# Location is always 'body' for this class
|
||||
self.target_field_location: str = "body"
|
||||
self.target_field_schema: Optional[Dict[str, Any]] = None
|
||||
self.json_schema_validator = json_schema_validator
|
||||
self.original_value_at_path: Any = None
|
||||
self.mismatched_value: Any = None
|
||||
self._try_find_mismatch_target_in_body()
|
||||
|
||||
def _try_find_mismatch_target_in_body(self):
|
||||
self.logger.info(f"[{self.id}] Initializing: Looking for a simple type field in request body for type mismatch test.")
|
||||
|
||||
body_schema_to_check = self._get_resolved_request_body_schema()
|
||||
|
||||
if body_schema_to_check:
|
||||
found_target = self._find_simple_type_field_in_schema(body_schema_to_check, "request body")
|
||||
if found_target:
|
||||
self.target_field_path, self.original_field_type, self.target_field_schema = found_target
|
||||
self.logger.info(f"[{self.id}] Target field for type mismatch (body): {'.'.join(map(str, self.target_field_path))}, Original Type: {self.original_field_type}")
|
||||
else:
|
||||
self.logger.info(f"[{self.id}] No suitable simple type field found in request body schema for type mismatch test.")
|
||||
else:
|
||||
self.logger.info(f"[{self.id}] No request body schema found for endpoint. Skipping type mismatch target search.")
|
||||
|
||||
if not self.target_field_path:
|
||||
self.logger.info(f"[{self.id}] Conclusion: No target field identified for request body type mismatch test.")
|
||||
|
||||
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
self.logger.debug(f"{self.id} is focused on request body, generate_query_params will not modify query parameters.")
|
||||
return current_query_params
|
||||
|
||||
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
|
||||
if not self.target_field_path or not self.original_field_type:
|
||||
self.logger.info(f"[{self.id}] No target field or original type identified for body type mismatch. Skipping body modification.")
|
||||
return current_body
|
||||
|
||||
self.logger.debug(f"[{self.id}] Preparing to modify request body for type mismatch. Target path: {self.target_field_path}, Original type: {self.original_field_type}")
|
||||
|
||||
# Get the original value at path for logging/context if needed (optional)
|
||||
# current_val_at_path, _ = schema_utils.util_get_value_at_path(current_body, self.target_field_path) # Assuming a get_value_at_path util exists or is added
|
||||
# For now, we don't strictly need original_value for generate_mismatched_value, but it takes it as an arg.
|
||||
|
||||
mismatched_value = schema_utils.generate_mismatched_value(
|
||||
original_type=self.original_field_type,
|
||||
original_value=None, # Placeholder, as current generate_mismatched_value doesn't use it heavily yet
|
||||
field_schema=self.target_field_schema,
|
||||
logger_param=self.logger
|
||||
)
|
||||
|
||||
self.logger.info(f"[{self.id}] Generated mismatched value '{mismatched_value}' for original type '{self.original_field_type}' at path '{'.'.join(map(str, self.target_field_path))}'.")
|
||||
|
||||
modified_body, success = schema_utils.util_set_value_at_path(
|
||||
data_container=current_body,
|
||||
path=self.target_field_path,
|
||||
new_value=mismatched_value
|
||||
)
|
||||
|
||||
if success:
|
||||
self.logger.debug(f"[{self.id}] Successfully set mismatched value at path using util_set_value_at_path.")
|
||||
return modified_body
|
||||
else:
|
||||
self.logger.error(f"[{self.id}] Failed to set mismatched value at path using util_set_value_at_path. Returning original body.")
|
||||
return current_body
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""验证API是否正确返回code=4001错误(状态码为200)。"""
|
||||
results = []
|
||||
if not self.target_field_path:
|
||||
self.logger.info(f"[{self.id}] Skipped type mismatch (body) validation: No target field was identified.")
|
||||
return [self.passed("跳过测试:在请求体中未找到合适的字段来测试类型不匹配。")]
|
||||
|
||||
expected_http_status_code = 200
|
||||
expected_business_error_code = -1
|
||||
|
||||
field_path_str = '.'.join(map(str, self.target_field_path))
|
||||
context_msg_prefix = f"当请求体字段 '{field_path_str}' 类型不匹配时, "
|
||||
|
||||
# 检查状态码
|
||||
if response_context.status_code != expected_http_status_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}。",
|
||||
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
|
||||
)]
|
||||
|
||||
# 检查业务错误码
|
||||
json_content = response_context.json_content
|
||||
if not isinstance(json_content, dict):
|
||||
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
|
||||
|
||||
actual_business_code = json_content.get("code")
|
||||
if actual_business_code != expected_business_error_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}。",
|
||||
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
|
||||
)]
|
||||
|
||||
return [self.passed(
|
||||
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}。"
|
||||
)]
|
||||
Vendored
+132
@@ -0,0 +1,132 @@
|
||||
from typing import Dict, Any, Optional, List, Tuple, Union
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import copy
|
||||
import logging
|
||||
from ddms_compliance_suite.utils import schema_utils
|
||||
|
||||
class TypeMismatchQueryParamCase(BaseAPITestCase):
|
||||
id = "TC-ERROR-4001-QUERY"
|
||||
name = "查询参数类型不匹配检查"
|
||||
description = "测试当发送的查询参数数据类型与API规范定义不符时,API是否按预期返回code=4001的错误。"
|
||||
severity = TestSeverity.MEDIUM
|
||||
tags = ["error-handling", "appendix-b", "4001", "query-parameters"]
|
||||
execution_order = 201 # Slightly after the combined one might have been
|
||||
|
||||
@classmethod
|
||||
def applies_to(cls, endpoint_spec: Dict[str, Any], **kwargs) -> bool:
|
||||
"""
|
||||
此测试用例仅适用于那些在API规范中定义了查询参数(parameters with 'in' == 'query')的端点。
|
||||
"""
|
||||
# 如果 'parameters' 字段不存在,则不适用
|
||||
if not endpoint_spec.get("parameters"):
|
||||
return False
|
||||
|
||||
# 遍历所有参数,如果发现任何一个参数的 "in" 字段是 "query",则此测试用例适用。
|
||||
return any(param.get("in") == "query" for param in endpoint_spec["parameters"])
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
self.target_field_path: Optional[List[str]] = None
|
||||
self.original_field_type: Optional[str] = None
|
||||
# Location is always 'query' for this class
|
||||
self.target_field_location: str = "query"
|
||||
self.target_field_schema: Optional[Dict[str, Any]] = None
|
||||
self.target_param_name: Optional[str] = None
|
||||
self.json_schema_validator = json_schema_validator
|
||||
self.original_value_at_path: Any = None
|
||||
self.mismatched_value: Any = None
|
||||
|
||||
# 调用新方法来查找目标字段
|
||||
self._try_find_mismatch_target_in_query()
|
||||
|
||||
def _try_find_mismatch_target_in_query(self):
|
||||
self.logger.info(f"[{self.id}] Initializing: Looking for a simple type query parameter for type mismatch test.")
|
||||
|
||||
found_target_param = self._find_first_simple_type_parameter(param_location="query")
|
||||
|
||||
if found_target_param:
|
||||
full_path, param_type, param_schema, top_level_param_name = found_target_param
|
||||
self.target_field_path = full_path
|
||||
self.original_field_type = param_type
|
||||
self.target_field_schema = param_schema
|
||||
self.target_param_name = top_level_param_name # Store the top-level parameter name
|
||||
self.logger.info(f"[{self.id}] Target for type mismatch (query): Param='{self.target_param_name}', Path='{'.'.join(map(str,self.target_field_path))}', Type='{self.original_field_type}'")
|
||||
else:
|
||||
self.logger.info(f"[{self.id}] No suitable simple type query parameter found for type mismatch test.")
|
||||
|
||||
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
|
||||
self.logger.debug(f"{self.id} is focused on query parameters, generate_request_body will not modify the body.")
|
||||
return current_body
|
||||
|
||||
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
if not self.target_field_path or not self.original_field_type:
|
||||
self.logger.info(f"[{self.id}] No target field or original type identified for query param type mismatch. Skipping query param modification.")
|
||||
return current_query_params
|
||||
|
||||
self.logger.debug(f"[{self.id}] Preparing to modify query params for type mismatch. Target path: {self.target_field_path}, Original type: {self.original_field_type}")
|
||||
|
||||
mismatched_value = schema_utils.generate_mismatched_value(
|
||||
original_type=self.original_field_type,
|
||||
original_value=None, # Placeholder
|
||||
field_schema=self.target_field_schema,
|
||||
logger_param=self.logger
|
||||
)
|
||||
|
||||
self.logger.info(f"[{self.id}] Generated mismatched value '{mismatched_value}' for original type '{self.original_field_type}' at query path '{'.'.join(map(str, self.target_field_path))}'.")
|
||||
|
||||
# Query parameters are typically a flat dictionary, but util_set_value_at_path can handle nested paths if needed (e.g. for object-style query params)
|
||||
modified_params, success = schema_utils.util_set_value_at_path(
|
||||
data_container=current_query_params,
|
||||
path=self.target_field_path, # Path might be like ['paramName'] or ['paramName', 'nestedKey']
|
||||
new_value=mismatched_value
|
||||
)
|
||||
|
||||
if success:
|
||||
self.logger.debug(f"[{self.id}] Successfully set mismatched value in query params using util_set_value_at_path.")
|
||||
return modified_params
|
||||
else:
|
||||
self.logger.error(f"[{self.id}] Failed to set mismatched value in query params using util_set_value_at_path. Returning original params.")
|
||||
return current_query_params
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
"""验证API是否正确返回code=4001错误(状态码为200)。"""
|
||||
results = []
|
||||
if not self.target_field_path:
|
||||
self.logger.info(f"[{self.id}] Skipped type mismatch (query) validation: No target query parameter was identified.")
|
||||
return [self.passed("跳过测试:在查询参数中未找到合适的字段来测试类型不匹配。")]
|
||||
|
||||
expected_http_status_code = 200
|
||||
expected_business_error_code = -1
|
||||
|
||||
context_param_identifier = self.target_param_name or '.'.join(map(str, self.target_field_path))
|
||||
context_msg_prefix = f"当查询参数 '{context_param_identifier}' (路径: '{'.'.join(map(str, self.target_field_path))}') 类型不匹配时, "
|
||||
|
||||
# 检查状态码
|
||||
if response_context.status_code != expected_http_status_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}。",
|
||||
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
|
||||
)]
|
||||
|
||||
# 检查业务错误码
|
||||
json_content = response_context.json_content
|
||||
if not isinstance(json_content, dict):
|
||||
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
|
||||
|
||||
actual_business_code = json_content.get("code")
|
||||
if actual_business_code != expected_business_error_code:
|
||||
return [self.failed(
|
||||
message=f"{context_msg_prefix}业务错误码应为 {expected_business_error_code},但实际为 {actual_business_code}。",
|
||||
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
|
||||
)]
|
||||
|
||||
return [self.passed(
|
||||
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 {expected_business_error_code}。"
|
||||
)]
|
||||
|
||||
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# ... existing code ...
|
||||
pass
|
||||
|
||||
return current_path_params
|
||||
@@ -0,0 +1 @@
|
||||
# This file marks the normative_spec directory as a Python package.
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
+67
@@ -0,0 +1,67 @@
|
||||
from typing import Dict, Any, Optional, List
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
|
||||
# class HTTPMethodUsageCase(BaseAPITestCase):
|
||||
# id = "TC-NORMATIVE-001"
|
||||
# name = "HTTP Method Usage Verification"
|
||||
# description = "验证API是否恰当使用HTTP方法(例如,GET用于检索,POST用于创建)。目前不测试对不支持方法的405响应。"
|
||||
# severity = TestSeverity.MEDIUM
|
||||
# tags = ["normative-spec", "http", "restful"]
|
||||
# execution_order = 110
|
||||
|
||||
# # 此测试通常适用。
|
||||
# # 检查对不支持方法的405响应会比较复杂,需要知道哪些方法对于每个路径是明确不支持的,
|
||||
# # 或者尝试所有其他方法,这在API规范中并不总是明确的。
|
||||
|
||||
# def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None):
|
||||
# super().__init__(endpoint_spec, global_api_spec, json_schema_validator)
|
||||
# self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
|
||||
|
||||
# def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
# results = []
|
||||
# method = request_context.method.upper()
|
||||
# status_code = response_context.status_code
|
||||
|
||||
# # 基于常见RESTful约定的基本检查
|
||||
# # 这些是通用指南,可能需要根据具体的API设计进行调整。
|
||||
|
||||
# if method == "GET":
|
||||
# if status_code // 100 == 2: # 成功的GET
|
||||
# results.append(self.passed(f"GET请求 {request_context.url} 返回了成功的状态码 {status_code}。"))
|
||||
# elif status_code == 404:
|
||||
# results.append(self.passed(f"GET请求 {request_context.url} 返回404,如果资源不存在则这是有效的响应。"))
|
||||
# # GET请求的其他状态码可能是错误或此处未覆盖的特定条件。
|
||||
|
||||
# elif method == "POST":
|
||||
# if status_code == 201: # 已创建
|
||||
# results.append(self.passed(f"POST请求 {request_context.url} 返回201 Created,符合资源创建的预期。"))
|
||||
# elif status_code == 200 or status_code == 202: # OK或已接受(例如,用于异步任务)
|
||||
# results.append(self.passed(f"POST请求 {request_context.url} 返回{status_code},这可以是有效的响应。"))
|
||||
# # 可添加对400(错误请求,例如payload无效)等的检查。
|
||||
|
||||
# elif method == "PUT":
|
||||
# if status_code == 200: # OK(已更新)
|
||||
# results.append(self.passed(f"PUT请求 {request_context.url} 返回200 OK,符合资源更新的预期。"))
|
||||
# elif status_code == 201: # 已创建(如果PUT在资源不存在时创建资源)
|
||||
# results.append(self.passed(f"PUT请求 {request_context.url} 返回201 Created,这可以是有效的响应。"))
|
||||
# elif status_code == 204: # 无内容(已更新,不返回响应体)
|
||||
# results.append(self.passed(f"PUT请求 {request_context.url} 返回204 No Content,这可以是有效的响应。"))
|
||||
# # 可添加对404(未找到,如果要更新的资源不存在,除非PUT会创建)的检查。
|
||||
|
||||
# elif method == "DELETE":
|
||||
# if status_code == 200 or status_code == 202 or status_code == 204: # OK、已接受或无内容
|
||||
# results.append(self.passed(f"DELETE请求 {request_context.url} 返回{status_code},表示成功删除。"))
|
||||
# # 可添加对404(未找到,如果要删除的资源不存在)的检查。
|
||||
|
||||
# # 405(方法不允许)检查的占位符 - 这比较复杂
|
||||
# # 要测试405,通常需要:
|
||||
# # 1. 知道哪些方法是此路径明确不允许的。
|
||||
# # 2. 或者,尝试使用其他常见方法(OPTIONS, PATCH等)发送请求,
|
||||
# # 如果这些方法未在此路径的规范中定义,则期望405。
|
||||
# # 这通常需要更具针对性的测试用例或不同的方法。
|
||||
# # self.logger.info("此通用测试用例未实现405(方法不允许)检查。")
|
||||
|
||||
# if not results: # 如果没有为该方法触发特定的验证
|
||||
# results.append(self.passed(f"针对 {method} {request_context.url} 的HTTP方法使用检查完成(基于通用约定未发现特定问题)。"))
|
||||
|
||||
# return results
|
||||
+247
@@ -0,0 +1,247 @@
|
||||
# import re
|
||||
# import json # 确保导入json
|
||||
# from typing import Dict, Any, Optional, List
|
||||
# from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext
|
||||
# # LLMService的导入路径需要根据您的项目结构确认
|
||||
# # 假设 LLMService 在 ddms_compliance_suite.llm_utils.llm_service
|
||||
# try:
|
||||
# from ddms_compliance_suite.llm_utils.llm_service import LLMService
|
||||
# except ImportError:
|
||||
# LLMService = None
|
||||
# # print("LLMService not found, PathVerbNounCheckCase will be skipped or limited.")
|
||||
|
||||
# class ComprehensiveURLCheckLLMCase(BaseAPITestCase):
|
||||
# id = "TC-NORMATIVE-URL-LLM-COMPREHENSIVE-001"
|
||||
# name = "综合URL规范与RESTful风格检查 (LLM)"
|
||||
# description = (
|
||||
# "使用LLM统一评估API路径是否符合以下规范:\n"
|
||||
# "1. 路径参数命名 (例如,全小写蛇形命名法)。\n"
|
||||
# "2. URL路径结构 (例如,/{领域}/{版本号}/资源类型)。\n"
|
||||
# "3. URL版本号嵌入 (例如,包含 /v1/)。\n"
|
||||
# "4. RESTful风格与可读性 (名词表示资源,HTTP方法表示动作,易理解性)。"
|
||||
# )
|
||||
# severity = TestSeverity.MEDIUM # 综合性检查,可能包含不同严重级别的问题
|
||||
# tags = ["normative-spec", "url", "restful", "llm", "readability", "naming-convention", "structure", "versioning", "static-check"]
|
||||
# execution_order = 60 # 更新执行顺序
|
||||
|
||||
# # 此测试用例可以覆盖所有路径,但其有效性依赖LLM
|
||||
# # applicable_methods = None
|
||||
# # applicable_paths_regex = None
|
||||
|
||||
# # 这个标志可以用来在测试用例级别控制是否实际调用LLM,即使全局LLM服务可用
|
||||
# # 如果您希望总是尝试(只要LLMService能初始化),可以不设置这个,或者在逻辑中检查 self.llm_service 是否存在
|
||||
# # use_llm_for_path_analysis: bool = True
|
||||
|
||||
# def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[LLMService] = None):
|
||||
# super().__init__(endpoint_spec, global_api_spec, json_schema_validator)
|
||||
# self.llm_service = llm_service # 存储注入的 LLMService 实例
|
||||
# if not self.llm_service:
|
||||
# self.logger.warning(f"LLMService 未注入或初始化失败,测试用例 {self.id} 将无法执行LLM路径分析。")
|
||||
|
||||
# def _get_llm_service_from_orchestrator(self) -> Optional[Any]:
|
||||
# # 在实际框架中,测试用例可能无法直接访问编排器来获取LLM服务。
|
||||
# # 这种依赖注入通常在测试用例实例化时或方法调用时处理。
|
||||
# # 此处为一个占位符,理想情况下APITestOrchestrator会将llm_service实例传给需要它的测试用例,
|
||||
# # 或测试用例通过某种服务定位器获取。
|
||||
# # 暂时我们假设,如果全局配置了LLM,它就能用。
|
||||
# # 真实的实现需要APITestOrchestrator在执行此测试用例前,将llm_service实例注入。
|
||||
# # 为了能运行,我们先返回None,并在下面逻辑中处理。
|
||||
# # 或者,修改 Orchestrator 将其注入到 self.global_api_spec 或 self.endpoint_spec (不推荐)
|
||||
# # 最好的方式是在 __init__ 中接收一个 llm_service: Optional[LLMService] 参数。
|
||||
# # 但这需要修改 BaseAPITestCase 和 APITestOrchestrator 的 __init__ 和调用逻辑。
|
||||
|
||||
# # 临时的解决方法:依赖 APITestOrchestrator 初始化时是否成功创建了 LLMService。
|
||||
# # 这仍然是一个间接的检查。一个更直接的方式是在Orchestrator执行此测试用例时传入。
|
||||
# if hasattr(self, '_orchestrator_llm_service_instance') and self._orchestrator_llm_service_instance:
|
||||
# return self._orchestrator_llm_service_instance
|
||||
|
||||
# # 如果没有明确注入,我们只能依赖全局LLMService是否被加载
|
||||
# if LLMService is not None:
|
||||
# # 这里不能直接实例化一个新的LLMService,因为它需要API Key等配置,这些配置在Orchestrator那里。
|
||||
# # 这个测试用例需要依赖Orchestrator来提供一个已经配置好的LLMService实例。
|
||||
# # 此处返回一个指示:如果LLM功能应该被使用,则需要Orchestrator提供服务。
|
||||
# return "NEEDS_INJECTION"
|
||||
# return None
|
||||
|
||||
# def _extract_path_param_names(self, path_template: str) -> List[str]:
|
||||
# """从路径模板中提取路径参数名称。例如 /users/{user_id}/items/{item_id} -> ['user_id', 'item_id']"""
|
||||
# return re.findall(r'\{([^}]+)\}', path_template)
|
||||
|
||||
# def validate_request_url(self, url: str, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
# results: List[ValidationResult] = []
|
||||
# path_template = self.endpoint_spec.get('path', '')
|
||||
# http_method = request_context.method.upper()
|
||||
# operation_id = self.endpoint_spec.get('operationId', self.endpoint_spec.get('title', '')) # 获取operationId或title
|
||||
|
||||
# if not self.llm_service:
|
||||
# results.append(ValidationResult(
|
||||
# passed=True, # 标记为通过以避免阻塞,但消息表明跳过
|
||||
# message=f"路径 '{path_template}' 的LLM综合URL检查已跳过:LLM服务不可用。",
|
||||
# details={"path_template": path_template, "http_method": http_method, "reason": "LLM Service not available or not injected."}
|
||||
# ))
|
||||
# self.logger.warning(f"LLM综合URL检查已跳过对路径 '{path_template}' 的检查:LLM服务不可用。")
|
||||
# return results
|
||||
|
||||
# path_param_names = self._extract_path_param_names(path_template)
|
||||
# path_params_str = ", ".join(path_param_names) if path_param_names else "无"
|
||||
|
||||
# # - 接口名称 (OperationId 或 Title): {operation_id if operation_id else '请你自己从路径模板中提取'}
|
||||
|
||||
# # 构建给LLM的Prompt,要求JSON输出
|
||||
# prompt_instruction = f"""
|
||||
# 请扮演一位资深的API设计评审员。我将提供一个API端点的路径模板、HTTP方法以及可能的接口名称。
|
||||
# 请根据以下石油行业API设计规范评估此API端点,并以严格的JSON格式返回您的评估结果。
|
||||
# JSON对象应包含一个名为 "assessments" 的键,其值为一个对象列表,每个对象代表对一个标准的评估,包含 "standard_name" (字符串), "is_compliant" (布尔值), 和 "reason" (字符串) 三个键。
|
||||
|
||||
# API端点信息:
|
||||
# - HTTP方法: {http_method}
|
||||
# - 路径模板: {path_template}
|
||||
# - 路径中提取的参数名: [{path_params_str}]
|
||||
|
||||
# 评估标准:
|
||||
|
||||
# 1. **接口名称规范 (接口名称需要你从路径模板中提取,一般是路径中除了参数名以外的最后的一个单词)**:
|
||||
# - 规则: 采用'动词+名词'结构,明确业务语义 (例如: GetWellLog, SubmitSeismicJob)。
|
||||
# - standard_name: "interface_naming_convention"
|
||||
|
||||
# 2. **HTTP方法使用规范**:
|
||||
# - 规则: 遵循RESTful规范:GET用于数据检索, POST用于创建资源, PUT用于更新资源, DELETE用于删除资源。
|
||||
# - standard_name: "http_method_usage"
|
||||
|
||||
# 3. **URL路径结构规范**:
|
||||
# - 规则: 格式为 `<前缀>/<专业领域>/v<版本号>/<资源类型>` (例如: /logging/v1.2/wells, /seismicprospecting/v1.0/datasets)。
|
||||
# - 前缀: 示例: /api/dms
|
||||
# - 专业领域: 专业领域示例: seismicprospecting, welllogging, reservoirevaluation
|
||||
# - 版本号: 语义化版本,例如 v1, v1.0, v2.1.3。
|
||||
# - 资源类型: 通常为名词复数。
|
||||
# - standard_name: "url_path_structure"
|
||||
|
||||
|
||||
# 4. **URL路径参数命名规范**:
|
||||
# - 规则: 路径参数(如果存在)必须使用全小写字母(可以是一个单词)或小写字母加下划线命名(这是多个单词的情况),并能反映资源的唯一标识 (例如: {{well_id}},还有{{version}},{{schema}}也是合规的,比一定非要{{version_id}})。
|
||||
# - standard_name: "url_path_parameter_naming"
|
||||
|
||||
|
||||
# 5. **资源命名规范 (在路径中)**:
|
||||
# - 规则: 资源集合应使用名词的复数形式表示 (例如 `/wells`, `/logs`);应优先使用石油行业的标准术语 (例如用 `trajectory` 而非 `path` 来表示井轨迹)。
|
||||
# - standard_name: "resource_naming_in_path"
|
||||
|
||||
|
||||
|
||||
|
||||
# 请确保您的输出是一个可以被 `json.loads()` 直接解析的JSON对象。
|
||||
# 例如:
|
||||
# {{
|
||||
# "assessments": [
|
||||
# {{
|
||||
# "standard_name": "interface_naming_convention",
|
||||
# "is_compliant": true,
|
||||
# "reason": "接口名称 'GetWellboreTrajectory' 符合动词+名词结构。"
|
||||
# }},
|
||||
# {{
|
||||
# "standard_name": "http_method_usage",
|
||||
# "is_compliant": true,
|
||||
# "reason": "GET方法用于检索资源,符合规范。"
|
||||
# }}
|
||||
# // ... 其他标准的评估 ...
|
||||
# ]
|
||||
# }}
|
||||
# """
|
||||
|
||||
# # 6. **路径可读性与整体RESTful风格**:
|
||||
# # - 规则: 路径整体是否具有良好的可读性、易于理解其功能,并且符合RESTful设计原则?(综合评估,可参考前面几点)
|
||||
# # - standard_name: "general_readability_and_restfulness"
|
||||
|
||||
# messages = [
|
||||
# {"role": "system", "content": "你是一位API设计评审专家,专注于评估API的URL规范性和RESTful风格。你的输出必须是严格的JSON格式。"},
|
||||
# {"role": "user", "content": prompt_instruction}
|
||||
# ]
|
||||
|
||||
# self.logger.info(f"向LLM发送请求,评估路径: {path_template} ({http_method})")
|
||||
# # 假设 _execute_chat_completion_request 支持 response_format={"type": "json_object"} (如果LLM API支持)
|
||||
# # 否则,我们需要解析文本输出。为简化,这里假设LLM会遵循JSON格式指令。
|
||||
# llm_response_str = self.llm_service._execute_chat_completion_request(
|
||||
# messages=messages,
|
||||
# max_tokens=1024, # 根据评估结果的复杂度调整
|
||||
# temperature=0.2 # 低温以获得更确定的、结构化的输出
|
||||
# )
|
||||
|
||||
# if not llm_response_str:
|
||||
# results.append(ValidationResult(
|
||||
# passed=False, # 执行失败
|
||||
# message=f"未能从LLM获取对路径 '{path_template}' 的评估。",
|
||||
# details={"path_template": path_template, "http_method": http_method, "reason": "LLM did not return a response."}
|
||||
# ))
|
||||
# self.logger.error(f"LLM对路径 '{path_template}' 的评估请求未返回任何内容。")
|
||||
# return results
|
||||
|
||||
# self.logger.debug(f"LLM对路径 '{path_template}' 的原始响应: {llm_response_str}")
|
||||
|
||||
# try:
|
||||
# # 尝试清理并解析LLM响应
|
||||
# # 有时LLM可能在JSON前后添加 "```json" 和 "```"
|
||||
# cleaned_response_str = llm_response_str.strip()
|
||||
# if cleaned_response_str.startswith("```json"):
|
||||
# cleaned_response_str = cleaned_response_str[7:]
|
||||
# if cleaned_response_str.endswith("```"):
|
||||
# cleaned_response_str = cleaned_response_str[:-3]
|
||||
|
||||
# llm_assessment_data = json.loads(cleaned_response_str)
|
||||
|
||||
# if "assessments" not in llm_assessment_data or not isinstance(llm_assessment_data["assessments"], list):
|
||||
# raise ValueError("LLM响应JSON中缺少 'assessments' 列表或格式不正确。")
|
||||
|
||||
# found_assessments = False
|
||||
# for assessment in llm_assessment_data["assessments"]:
|
||||
# standard_name = assessment.get("standard_name", "未知标准")
|
||||
# is_compliant = assessment.get("is_compliant", False)
|
||||
# reason = assessment.get("reason", "LLM未提供原因。")
|
||||
# found_assessments = True
|
||||
|
||||
# results.append(ValidationResult(
|
||||
# passed=is_compliant,
|
||||
# message=f"LLM评估 - {standard_name}: {reason}",
|
||||
# details={
|
||||
# "standard_name": standard_name,
|
||||
# "is_compliant_by_llm": is_compliant,
|
||||
# "llm_reason": reason,
|
||||
# "path_template": path_template,
|
||||
# "http_method": http_method
|
||||
# }
|
||||
# ))
|
||||
# log_level = self.logger.info if is_compliant else self.logger.warning
|
||||
# log_level(f"LLM评估 - 标准 '{standard_name}' for '{path_template}': {'符合' if is_compliant else '不符合'}。原因: {reason}")
|
||||
|
||||
# if not found_assessments:
|
||||
# results.append(ValidationResult(
|
||||
# passed=False,
|
||||
# message=f"LLM返回的评估结果中不包含任何有效的评估项。",
|
||||
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str}
|
||||
# ))
|
||||
|
||||
|
||||
# except json.JSONDecodeError as e_json:
|
||||
# results.append(ValidationResult(
|
||||
# passed=False, # 执行失败
|
||||
# message=f"无法将LLM对路径 '{path_template}' 的评估响应解析为JSON: {e_json}",
|
||||
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str, "error": str(e_json)}
|
||||
# ))
|
||||
# self.logger.error(f"LLM对路径 '{path_template}' 的响应JSON解析失败: {e_json}. Raw response: {llm_response_str}")
|
||||
# except ValueError as e_val: # 自定义错误,如缺少 'assessments'
|
||||
# results.append(ValidationResult(
|
||||
# passed=False, # 执行失败
|
||||
# message=f"LLM对路径 '{path_template}' 的评估响应JSON结构不符合预期: {e_val}",
|
||||
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str, "error": str(e_val)}
|
||||
# ))
|
||||
# self.logger.error(f"LLM对路径 '{path_template}' 的响应JSON结构错误: {e_val}. Raw response: {llm_response_str}")
|
||||
# except Exception as e_generic:
|
||||
# results.append(ValidationResult(
|
||||
# passed=False, # 执行失败
|
||||
# message=f"处理LLM对路径 '{path_template}' 的评估响应时发生未知错误: {e_generic}",
|
||||
# details={"path_template": path_template, "http_method": http_method, "raw_llm_response": llm_response_str, "error": str(e_generic)}
|
||||
# ))
|
||||
# self.logger.error(f"处理LLM对路径 '{path_template}' 的响应时发生未知错误: {e_generic}", exc_info=True)
|
||||
|
||||
# return results
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
# This file marks the security directory as a Python package.
|
||||
BIN
Binary file not shown.
+84
@@ -0,0 +1,84 @@
|
||||
from typing import Dict, Any, Optional, List
|
||||
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
|
||||
import urllib.parse
|
||||
|
||||
class HTTPSMandatoryCase(BaseAPITestCase):
|
||||
id = "TC-SECURITY-001"
|
||||
name = "HTTPS 协议强制性检查"
|
||||
description = "验证API端点是否通过HTTPS提供服务,以及HTTP请求是否被拒绝或重定向到HTTPS。"
|
||||
severity = TestSeverity.HIGH
|
||||
tags = ["security", "https", "transport-security"]
|
||||
execution_order = 120
|
||||
|
||||
# 此测试会修改URL为HTTP,应适用于大多数端点。
|
||||
|
||||
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
|
||||
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
|
||||
self.logger.info(f"测试用例 '{self.id}' 已为端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
|
||||
|
||||
def modify_request_url(self, current_url: str) -> str:
|
||||
parsed_url = urllib.parse.urlparse(current_url)
|
||||
if parsed_url.scheme.lower() == "https":
|
||||
# 将 https 替换为 http
|
||||
modified_url = parsed_url._replace(scheme="http").geturl()
|
||||
self.logger.info(f"为进行HTTPS检查修改URL:原始 '{current_url}', 修改为 '{modified_url}'")
|
||||
return modified_url
|
||||
else:
|
||||
self.logger.warning(f"原始URL '{current_url}' 不是HTTPS。跳过此测试用例的URL修改。")
|
||||
# 如果原始URL不是HTTPS,此测试可能无效,
|
||||
# 或者暗示基础URL本身可能未正确配置以进行HTTPS测试。
|
||||
return current_url # 如果不是HTTPS则返回原始URL
|
||||
|
||||
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
|
||||
results = []
|
||||
status_code = response_context.status_code
|
||||
# request_context.url 是调用 modify_request_url 之后,APICaller实际发送的URL
|
||||
request_url_scheme = urllib.parse.urlparse(request_context.url).scheme
|
||||
|
||||
# 检查URL是否确实被此测试的钩子修改为了HTTP
|
||||
if request_url_scheme.lower() != "http":
|
||||
results.append(self.passed(
|
||||
message=f"测试已跳过,因为发送的URL已经是 {request_url_scheme.upper()}(可能是由于原始基础URL非HTTPS或测试设置问题)。"
|
||||
))
|
||||
self.logger.info("HTTPS强制性检查已跳过,因为有效URL不是HTTP。")
|
||||
return results
|
||||
|
||||
# 如果请求是通过HTTP发出的,我们期望几种结果:
|
||||
# 1. 拒绝(例如400、403,或连接被拒绝 - 尽管APICaller可能在此之前处理连接拒绝)
|
||||
# 2. 重定向到HTTPS(例如301、302、307、308,并带有指向HTTPS的Location头)
|
||||
|
||||
if status_code in [301, 302, 307, 308]: # 重定向状态码
|
||||
location_header = response_context.headers.get("Location")
|
||||
if location_header and urllib.parse.urlparse(location_header).scheme.lower() == "httpss":
|
||||
results.append(self.passed(
|
||||
message=f"对 {request_context.url} 的HTTP请求被正确重定向到HTTPS ({location_header}),状态码 {status_code}。"
|
||||
))
|
||||
self.logger.info(f"HTTP被正确重定向到HTTPS: {location_header}")
|
||||
else:
|
||||
results.append(self.failed(
|
||||
message=f"对 {request_context.url} 的HTTP请求被重定向(状态码 {status_code}),但Location头 '{location_header}' 未指向HTTPS URL。",
|
||||
details={"status_code": status_code, "location_header": location_header}
|
||||
))
|
||||
self.logger.warning(f"HTTP被重定向,状态码 {status_code},但Location '{location_header}' 不是HTTPS。")
|
||||
elif status_code // 100 == 4: # 客户端错误(例如400错误请求,403禁止访问)
|
||||
results.append(self.passed(
|
||||
message=f"对 {request_context.url} 的HTTP请求被客户端错误(状态码 {status_code})拒绝,表明不允许HTTP访问。"
|
||||
))
|
||||
self.logger.info(f"HTTP请求被客户端错误 {status_code} 拒绝。")
|
||||
elif status_code // 100 == 2: # 通过HTTP成功返回2xx响应
|
||||
results.append(self.failed(
|
||||
message=f"API通过HTTP ({request_context.url}) 响应了成功的状态码 {status_code},这违反了HTTPS强制策略。",
|
||||
details={"status_code": status_code}
|
||||
))
|
||||
self.logger.error(f"安全漏洞:API允许通过HTTP成功响应 ({status_code})。")
|
||||
else:
|
||||
# 其他状态码(例如5xx)可能表示与HTTPS强制执行无关的服务器错误,
|
||||
# 或者连接在更底层被拒绝(APICaller可能会抛出异常)。
|
||||
results.append(ValidationResult(
|
||||
passed=False, # 或True,取决于严格程度 - 5xx不是通过,但不一定是HTTPS失败
|
||||
message=f"对 {request_context.url} 的HTTP请求返回了意外的状态码 {status_code}。需要手动调查。",
|
||||
details={"status_code": status_code, "response_text_sample": (response_context.text_content or "")[:200]}
|
||||
))
|
||||
self.logger.warning(f"HTTP请求返回意外状态码 {status_code}。潜在问题或服务器错误。")
|
||||
|
||||
return results
|
||||
Reference in New Issue
Block a user