This commit is contained in:
gongwenxin
2025-06-16 14:49:49 +08:00
parent adc1a0053f
commit df90a5377f
210 changed files with 323584 additions and 12804 deletions
@@ -0,0 +1,164 @@
from typing import Dict, Any, Optional, List, Union, Tuple
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import logging
from ddms_compliance_suite.utils import schema_utils
import random
import string
class InvalidEnumValueCase(BaseAPITestCase):
id = "TC-ERROR-4006"
name = "Error Code 4006 - Invalid Enum Value Validation"
description = "测试当发送的参数值不在指定的枚举范围内时,API是否按预期返回code=4006的错误(状态码应为200)。"
severity = TestSeverity.MEDIUM
tags = ["error-handling", "appendix-b", "4006", "invalid-enum"]
execution_order = 204 # 在数值越界之后执行
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.setLevel(logging.INFO)
self.target_field_location: Optional[str] = None
self.target_field_path: Optional[List[Union[str, int]]] = None
self.target_field_name: Optional[str] = None
self.valid_enums: Optional[List[str]] = None
self.invalid_enum_value: Optional[str] = None
self._initialize_test_case()
def _find_first_enum_field(self) -> Optional[Tuple[str, List[Union[str, int]], str, List[str], str]]:
"""
按优先级(body -> query -> path -> header)查找第一个具有枚举值限制的字段。
"""
self.logger.info(f"[{self.id}] Searching for a field with enum constraints...")
# 1. 检查请求体 (Body)
body_schema = self._get_resolved_request_body_schema()
if body_schema:
self.logger.debug(f"[{self.id}] Checking for enum field in request body.")
result = schema_utils.util_find_enum_field_recursive(body_schema, [], self.logger)
if result:
path, valid_enums, desc = result
name = path[-1] if path else "body_root"
self.logger.info(f"[{self.id}] Found enum field in BODY at path: {'.'.join(map(str, path))}")
return "body", path, name, valid_enums, desc
# 2. 检查查询参数, 路径参数, 请求头
for location in ["query", "path", "header"]:
if 'parameters' in self.endpoint_spec:
self.logger.debug(f"[{self.id}] Checking for enum field in {location} parameters.")
for param in self.endpoint_spec['parameters']:
if param.get('in') == location:
param_desc = param.get('description', '')
param_name = param.get('name')
valid_enums = schema_utils.util_extract_enum_from_description(param_desc, self.logger)
if valid_enums:
self.logger.info(f"[{self.id}] Found enum field in {location.upper()}: '{param_name}'")
return location, [param_name], param_name, valid_enums, param_desc
self.logger.info(f"[{self.id}] No field with enum constraints found in any location.")
return None
def _initialize_test_case(self):
"""初始化测试用例。"""
found_field = self._find_first_enum_field()
if found_field:
location, path, name, valid_enums, description = found_field
self.target_field_location = location
self.target_field_path = path
self.target_field_name = name
self.valid_enums = valid_enums
self.logger.info(
f"[{self.id}] Target found in '{location}' at path '{'.'.join(map(str, path))}'. "
f"Valid enums: {valid_enums}, Desc: '{description}'"
)
self._set_invalid_enum_value()
else:
self.logger.info(f"[{self.id}] No suitable enum field found for this endpoint.")
def _set_invalid_enum_value(self):
"""生成一个非法的枚举值。"""
random_suffix = ''.join(random.choices(string.ascii_lowercase + string.digits, k=6))
self.invalid_enum_value = f"invalid_enum_{random_suffix}"
self.logger.info(f"[{self.id}] Setting invalid enum value to '{self.invalid_enum_value}' (valid are {self.valid_enums}).")
def _modify_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
"""通用函数,用于修改参数字典中的值。"""
if self.invalid_enum_value is None or not self.target_field_path:
return params
modified_params, success = schema_utils.util_set_value_at_path(
data_container=params,
path=self.target_field_path,
new_value=self.invalid_enum_value
)
if success:
self.logger.debug(f"[{self.id}] Successfully injected invalid enum value into parameters.")
return modified_params
self.logger.error(f"[{self.id}] Failed to set invalid enum value in parameters.")
return params
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "query":
return self._modify_params(current_query_params)
return current_query_params
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "path":
return self._modify_params(current_path_params)
return current_path_params
def generate_headers(self, current_headers: Dict[str, str]) -> Dict[str, str]:
if self.target_field_location == "header":
return self._modify_params(current_headers)
return current_headers
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
if self.target_field_location == "body":
return self._modify_params(current_body)
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4006错误(状态码为200)。"""
results = []
if self.invalid_enum_value is None:
return [self.passed("跳过测试:未找到具有明确枚举值限制的字段。")]
expected_http_status_code = 200
expected_business_error_code = "4006"
context_msg_prefix = (
f"{self.target_field_location} 字段 '{self.target_field_name}' "
f"值为 '{self.invalid_enum_value}' (合法值为: {self.valid_enums}) 时, "
)
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if str(actual_business_code) != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 '{expected_business_error_code}',但实际为 '{actual_business_code}'",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 '{expected_business_error_code}'"
)]
@@ -6,7 +6,7 @@ from ddms_compliance_suite.utils import schema_utils # Keep this import for util
class MissingRequiredFieldBodyCase(BaseAPITestCase):
id = "TC-ERROR-4003-BODY"
name = "Error Code 4003 - Missing Required Request Body Field Validation"
description = "测试当请求体中缺少API规范定义的必填字段时,API是否按预期返回类似4003的错误(或通用400错误)"
description = "测试当请求体中缺少API规范定义的必填字段时,API是否按预期返回类似4003的错误。"
severity = TestSeverity.HIGH
tags = ["error-handling", "appendix-b", "4003", "required-fields", "request-body"]
execution_order = 210
@@ -53,70 +53,38 @@ class MissingRequiredFieldBodyCase(BaseAPITestCase):
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4003错误(状态码为200)。"""
results = []
if not self.removed_field_path:
results.append(self.passed("跳过测试:在API规范中未找到合适的必填请求体字段用于移除测试。"))
self.logger.info("由于未识别到可移除的必填请求体字段,跳过此测试用例的验证。")
return results
status_code = response_context.status_code
json_content = response_context.json_content
expected_http_status_codes = [400, 422] # Common client error codes
# specific_business_error_code 是此测试用例期望的特定业务错误码,例如 "4003"
# 这个值可以根据实际 API 的错误码约定在子类中调整或作为参数传入
specific_business_error_code = "4003"
error_code_field_in_body = "code" # 响应体中业务错误码的字段名
expected_http_status_code = 200
expected_business_error_code = "4003"
removed_field_str = '.'.join(map(str, self.removed_field_path))
context_msg_prefix = f"当移除必填请求体字段 '{removed_field_str}'"
context_msg_prefix = f"当移除必填请求体字段 '{removed_field_str}', "
http_status_ok = status_code in expected_http_status_codes
business_code_ok = False
is_4xx_error = 400 <= status_code <= 499
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
if json_content and isinstance(json_content, dict):
body_code = json_content.get(error_code_field_in_body)
if body_code is not None and str(body_code) == specific_business_error_code:
business_code_ok = True
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
if http_status_ok:
if business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code} 并且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} and business code '{specific_business_error_code}' match. (Removed field: body.{removed_field_str})")
else:
# HTTP status is OK, but business code is not what we specifically hoped for (or not present).
# Still considered a pass because the primary condition (HTTP status) is met.
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code}. "
f"响应体中的业务错误码 (\'{error_code_field_in_body}\': \'{json_content.get(error_code_field_in_body)}\') 与特定期望 \'{specific_business_error_code}\' 不符或未找到,但HTTP状态码正确。"
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} is correct. Business code mismatch/missing (Expected: \'{specific_business_error_code}\', Got: \'{json_content.get(error_code_field_in_body)}\'). (Removed field: body.{removed_field_str})")
elif business_code_ok:
# HTTP status was not in the primary list, but it's a 4xx and the business code matches.
results.append(self.passed(
f"{context_msg_prefix}API响应了状态码 {status_code} (非主要预期HTTP状态 {expected_http_status_codes},但为4xx客户端错误), "
f"且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed (Fallback). HTTP status {status_code} (4xx) with matching business code '{specific_business_error_code}'. (Removed field: body.{removed_field_str})")
else:
# Neither condition for passing was met.
fail_message = f"{context_msg_prefix}期望API返回状态码在 {expected_http_status_codes} 中,或返回4xx客户端错误且业务码为 '{specific_business_error_code}'."
fail_message += f" 实际收到状态码 {status_code}."
if json_content and isinstance(json_content, dict):
fail_message += f" 响应体中的业务码 (\'{error_code_field_in_body}\') 为 \'{json_content.get(error_code_field_in_body)}\'."
elif json_content:
fail_message += " 响应体不是一个JSON对象."
else:
fail_message += " 响应体为空或非JSON."
actual_business_code = json_content.get("code")
if str(actual_business_code) != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 '{expected_business_error_code}',但实际为 '{actual_business_code}'",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
results.append(self.failed(
message=fail_message,
details={"status_code": status_code, "response_body": json_content, "expected_http_status_codes": expected_http_status_codes, "expected_business_code": specific_business_error_code, "removed_field": f"body.{removed_field_str}"}
))
self.logger.warning(f"{self.id}: Failed. {fail_message} (Removed field: body.{removed_field_str})")
return results
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 '{expected_business_error_code}'"
)]
@@ -5,7 +5,7 @@ import copy
class MissingRequiredFieldQueryCase(BaseAPITestCase):
id = "TC-ERROR-4003-QUERY"
name = "Error Code 4003 - Missing Required Query Parameter Validation"
description = "测试当请求中缺少API规范定义的必填查询参数时,API是否按预期返回类似4003的错误(或通用400错误)"
description = "测试当请求中缺少API规范定义的必填查询参数时,API是否按预期返回类似4003的错误。"
severity = TestSeverity.HIGH
tags = ["error-handling", "appendix-b", "4003", "required-fields", "query-parameters"]
execution_order = 211 # After body, before original combined one might have been
@@ -43,6 +43,7 @@ class MissingRequiredFieldQueryCase(BaseAPITestCase):
return current_query_params
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4003错误(状态码为200)。"""
results = []
if not self.target_param_name:
@@ -50,57 +51,30 @@ class MissingRequiredFieldQueryCase(BaseAPITestCase):
self.logger.info("由于未识别到可移除的必填查询参数,跳过此测试用例。")
return results
status_code = response_context.status_code
expected_http_status_code = 200
expected_business_error_code = "4003"
context_msg_prefix = f"当移除必填查询参数 '{self.target_param_name}' 时, "
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
expected_http_status_codes = [400, 422]
specific_business_error_code = "4003"
error_code_field_in_body = "code"
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
context_msg_prefix = f"当移除必填查询参数 '{self.target_param_name}' 时,"
actual_business_code = json_content.get("code")
if str(actual_business_code) != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 '{expected_business_error_code}',但实际为 '{actual_business_code}'",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
http_status_ok = status_code in expected_http_status_codes
business_code_ok = False
is_4xx_error = 400 <= status_code <= 499
if json_content and isinstance(json_content, dict):
body_code = json_content.get(error_code_field_in_body)
if body_code is not None and str(body_code) == specific_business_error_code:
business_code_ok = True
if http_status_ok:
if business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code} 并且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} and business code '{specific_business_error_code}' match. (Removed query param: {self.target_param_name})")
else:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code}. "
f"响应体中的业务错误码 (\'{error_code_field_in_body}\': \'{json_content.get(error_code_field_in_body)}\') 与特定期望 \'{specific_business_error_code}\' 不符或未找到,但HTTP状态码正确。"
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} is correct. Business code mismatch/missing (Expected: \'{specific_business_error_code}\', Got: \'{json_content.get(error_code_field_in_body)}\'). (Removed query param: {self.target_param_name})")
elif business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了状态码 {status_code} (非主要预期HTTP状态 {expected_http_status_codes},但为4xx客户端错误), "
f"且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed (Fallback). HTTP status {status_code} (4xx) with matching business code '{specific_business_error_code}'. (Removed query param: {self.target_param_name})")
else:
fail_message = f"{context_msg_prefix}期望API返回状态码在 {expected_http_status_codes} 中,或返回4xx客户端错误且业务码为 '{specific_business_error_code}'."
fail_message += f" 实际收到状态码 {status_code}."
if json_content and isinstance(json_content, dict):
fail_message += f" 响应体中的业务码 (\'{error_code_field_in_body}\') 为 \'{json_content.get(error_code_field_in_body)}\'."
elif json_content:
fail_message += " 响应体不是一个JSON对象."
else:
fail_message += " 响应体为空或非JSON."
results.append(self.failed(
message=fail_message,
details={"status_code": status_code, "response_body": json_content, "expected_http_status_codes": expected_http_status_codes, "expected_business_code": specific_business_error_code, "removed_param": f"query.{self.target_param_name}"}
))
self.logger.warning(f"{self.id}: Failed. {fail_message} (Removed query param: {self.target_param_name})")
return results
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 '{expected_business_error_code}'"
)]
@@ -0,0 +1,193 @@
from typing import Dict, Any, Optional, List, Union, Tuple
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import logging
from ddms_compliance_suite.utils import schema_utils
class NumberOutOfRangeCase(BaseAPITestCase):
id = "TC-ERROR-4002"
name = "Error Code 4002 - Number Value Out of Range Validation"
description = "测试当发送的数值参数超出范围限制时,API是否按预期返回code=4002的错误(状态码应为200)。"
severity = TestSeverity.MEDIUM
tags = ["error-handling", "appendix-b", "4002", "out-of-range"]
execution_order = 203 # 在类型不匹配测试之后执行
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
self.logger.setLevel(logging.DEBUG)
self.target_field_location: Optional[str] = None
self.target_field_path: Optional[List[Union[str, int]]] = None
self.target_field_schema: Optional[Dict[str, Any]] = None
self.target_field_name: Optional[str] = None
self.target_field_type: Optional[str] = None
self.min_value: Optional[float] = None
self.max_value: Optional[float] = None
self.out_of_range_value: Optional[Union[int, float]] = None
self._initialize_test_case()
def _find_first_ranged_numeric_field(self) -> Optional[Tuple[str, List[Union[str, int]], Dict[str, Any], str, Optional[str], Optional[float], Optional[float], str]]:
"""
按优先级(body -> query -> path -> header)查找第一个具有范围限制的数值字段。
"""
self.logger.info(f"[{self.id}] Searching for a ranged numeric field...")
# 1. 检查请求体 (Body)
body_schema = self._get_resolved_request_body_schema()
if body_schema:
self.logger.debug(f"[{self.id}] Checking for ranged numeric field in request body.")
result = schema_utils.util_find_ranged_field_recursive(body_schema, [], self.logger)
if result:
path, schema, type, min_val, max_val, desc = result
name = path[-1] if path else "body_root"
self.logger.info(f"[{self.id}] Found ranged numeric field in BODY at path: {'.'.join(map(str, path))}")
return "body", path, schema, name, type, min_val, max_val, desc
# 2. 检查查询参数, 路径参数, 请求头
for location in ["query", "path", "header"]:
if 'parameters' in self.endpoint_spec:
self.logger.debug(f"[{self.id}] Checking for ranged numeric field in {location} parameters.")
for param in self.endpoint_spec['parameters']:
if param.get('in') == location:
param_schema = param.get('schema', {})
param_desc = param.get('description', '')
param_name = param.get('name')
min_val, max_val = schema_utils.util_extract_range_from_description(param_desc, self.logger)
if min_val is None and max_val is None:
min_val = param_schema.get('minimum')
max_val = param_schema.get('maximum')
if min_val is not None or max_val is not None:
param_type = param_schema.get('type')
self.logger.info(f"[{self.id}] Found ranged numeric field in {location.upper()}: '{param_name}'")
return location, [param_name], param_schema, param_name, param_type, min_val, max_val, param_desc
self.logger.info(f"[{self.id}] No ranged numeric field found in any location.")
return None
def _initialize_test_case(self):
"""使用框架辅助方法初始化测试用例。"""
found_field = self._find_first_ranged_numeric_field()
if found_field:
location, path, schema, name, field_type, min_val, max_val, description = found_field
self.target_field_location = location
self.target_field_path = path
self.target_field_schema = schema
self.target_field_name = name
# 如果'type'未定义,我们根据是否存在小数来推断
self.target_field_type = field_type if field_type else ('number' if isinstance(min_val, float) or isinstance(max_val, float) else 'integer')
self.min_value = min_val
self.max_value = max_val
self.logger.info(
f"[{self.id}] Target found in '{location}' at path '{'.'.join(map(str, path))}'. "
f"Type: {self.target_field_type}, Min: {min_val}, Max: {max_val}, Desc: '{description}'"
)
self._set_out_of_range_value()
else:
self.logger.info(f"[{self.id}] No suitable ranged numeric field found for this endpoint.")
def _set_out_of_range_value(self):
"""根据提取的范围信息设置越界值。"""
# 优先选择超过最大值
if self.max_value is not None:
if self.target_field_type == 'integer':
self.out_of_range_value = int(self.max_value) + 1
else:
self.out_of_range_value = self.max_value + 1.0
self.logger.info(f"[{self.id}] Setting out-of-range value to {self.out_of_range_value} (max is {self.max_value}).")
return
# 其次选择小于最小值
if self.min_value is not None:
if self.target_field_type == 'integer':
self.out_of_range_value = int(self.min_value) - 1
else:
self.out_of_range_value = self.min_value - 1.0
self.logger.info(f"[{self.id}] Setting out-of-range value to {self.out_of_range_value} (min is {self.min_value}).")
def _modify_params(self, params: Dict[str, Any]) -> Dict[str, Any]:
"""通用函数,用于修改参数字典中的值。"""
if self.out_of_range_value is None or not self.target_field_path:
return params
modified_params, success = schema_utils.util_set_value_at_path(
data_container=params,
path=self.target_field_path,
new_value=self.out_of_range_value
)
if success:
self.logger.debug(f"[{self.id}] Successfully injected out-of-range value into parameters.")
return modified_params
self.logger.error(f"[{self.id}] Failed to set out-of-range value in parameters.")
return params
def generate_query_params(self, current_query_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "query":
return self._modify_params(current_query_params)
return current_query_params
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
if self.target_field_location == "path":
return self._modify_params(current_path_params)
return current_path_params
def generate_headers(self, current_headers: Dict[str, str]) -> Dict[str, str]:
if self.target_field_location == "header":
# Header values must be strings
original_value = self.out_of_range_value
self.out_of_range_value = str(self.out_of_range_value)
headers = self._modify_params(current_headers)
self.out_of_range_value = original_value # revert for logging
return headers
return current_headers
def generate_request_body(self, current_body: Optional[Any]) -> Optional[Any]:
if self.target_field_location == "body":
return self._modify_params(current_body)
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4002错误(状态码为200)。"""
results = []
if self.out_of_range_value is None:
return [self.passed("跳过测试:未找到具有明确范围限制的数值字段。")]
expected_http_status_code = 200
expected_business_error_code = "4002"
context_msg_prefix = (
f"{self.target_field_location} 字段 '{self.target_field_name}' "
f"值为 {self.out_of_range_value} (超出范围: min={self.min_value}, max={self.max_value}) 时, "
)
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
actual_business_code = json_content.get("code")
if str(actual_business_code) != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 '{expected_business_error_code}',但实际为 '{actual_business_code}'",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 '{expected_business_error_code}'"
)]
@@ -81,64 +81,37 @@ class TypeMismatchBodyCase(BaseAPITestCase):
return current_body
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4001错误(状态码为200)。"""
results = []
if not self.target_field_path:
self.logger.info(f"[{self.id}] Skipped type mismatch (body) validation: No target field was identified.")
return [self.passed("跳过测试:在请求体中未找到合适的字段来测试类型不匹配。")]
status_code = response_context.status_code
json_content = response_context.json_content
expected_http_status_codes = [400, 422] # Common client error codes for type issues
# specific_business_error_code 是此测试用例期望的特定业务错误码,例如 "4001"
specific_business_error_code = "4001"
error_code_field_in_body = "code" # 响应体中业务错误码的字段名
expected_http_status_code = 200
expected_business_error_code = "4001"
field_path_str = '.'.join(map(str, self.target_field_path))
context_msg_prefix = f"当请求体字段 '{field_path_str}' 类型不匹配时"
context_msg_prefix = f"当请求体字段 '{field_path_str}' 类型不匹配时, "
http_status_ok = status_code in expected_http_status_codes
business_code_ok = False
is_4xx_error = 400 <= status_code <= 499
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
if json_content and isinstance(json_content, dict):
body_code = json_content.get(error_code_field_in_body)
if body_code is not None and str(body_code) == specific_business_error_code:
business_code_ok = True
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
if http_status_ok:
if business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code} 并且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} and business code '{specific_business_error_code}' match. (Field: body.{field_path_str})")
else:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code}. "
f"响应体中的业务错误码 (\'{error_code_field_in_body}\': \'{json_content.get(error_code_field_in_body)}\') 与特定期望 \'{specific_business_error_code}\' 不符或未找到,但HTTP状态码正确。"
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} is correct. Business code mismatch/missing (Expected: \'{specific_business_error_code}\', Got: \'{json_content.get(error_code_field_in_body)}\'). (Field: body.{field_path_str})")
elif business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了状态码 {status_code} (非主要预期HTTP状态 {expected_http_status_codes},但为4xx客户端错误), "
f"且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed (Fallback). HTTP status {status_code} (4xx) with matching business code '{specific_business_error_code}'. (Field: body.{field_path_str})")
else:
fail_message = f"{context_msg_prefix}期望API返回状态码在 {expected_http_status_codes} 中,或返回4xx客户端错误且业务码为 '{specific_business_error_code}'."
fail_message += f" 实际收到状态码 {status_code}."
if json_content and isinstance(json_content, dict):
fail_message += f" 响应体中的业务码 (\'{error_code_field_in_body}\') 为 \'{json_content.get(error_code_field_in_body)}\'."
elif json_content:
fail_message += " 响应体不是一个JSON对象."
else:
fail_message += " 响应体为空或非JSON."
actual_business_code = json_content.get("code")
if str(actual_business_code) != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 '{expected_business_error_code}',但实际为 '{actual_business_code}'",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
results.append(self.failed(
message=fail_message,
details={"status_code": status_code, "response_body": json_content, "expected_http_status_codes": expected_http_status_codes, "expected_business_code": specific_business_error_code, "mismatched_field": f"body.{field_path_str}"}
))
self.logger.warning(f"{self.id}: Failed. {fail_message} (Field: body.{field_path_str})")
return results
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 '{expected_business_error_code}'"
)]
@@ -78,67 +78,40 @@ class TypeMismatchQueryParamCase(BaseAPITestCase):
return current_query_params
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API是否正确返回code=4001错误(状态码为200)。"""
results = []
if not self.target_field_path:
self.logger.info(f"[{self.id}] Skipped type mismatch (query) validation: No target query parameter was identified.")
return [self.passed("跳过测试:在查询参数中未找到合适的字段来测试类型不匹配。")]
status_code = response_context.status_code
json_content = response_context.json_content
expected_http_status_codes = [400, 422]
specific_business_error_code = "4001"
error_code_field_in_body = "code"
# Use self.target_param_name for a clearer context message if a top-level param was identified
expected_http_status_code = 200
expected_business_error_code = "4001"
context_param_identifier = self.target_param_name or '.'.join(map(str, self.target_field_path))
context_msg_prefix = f"当查询参数 '{context_param_identifier}' (路径: '{'.'.join(map(str, self.target_field_path))}') 类型不匹配时"
context_msg_prefix = f"当查询参数 '{context_param_identifier}' (路径: '{'.'.join(map(str, self.target_field_path))}') 类型不匹配时, "
http_status_ok = status_code in expected_http_status_codes
business_code_ok = False
is_4xx_error = 400 <= status_code <= 499
# 检查状态码
if response_context.status_code != expected_http_status_code:
return [self.failed(
message=f"{context_msg_prefix}API应返回状态码 {expected_http_status_code},但实际为 {response_context.status_code}",
details={"expected_status": expected_http_status_code, "actual_status": response_context.status_code}
)]
if json_content and isinstance(json_content, dict):
body_code = json_content.get(error_code_field_in_body)
if body_code is not None and str(body_code) == specific_business_error_code:
business_code_ok = True
# 检查业务错误码
json_content = response_context.json_content
if not isinstance(json_content, dict):
return [self.failed(f"{context_msg_prefix}API响应体不是一个有效的JSON对象。")]
if http_status_ok:
if business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code} 并且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} and business code '{specific_business_error_code}' match. (Query param: {context_param_identifier})")
else:
results.append(self.passed(
f"{context_msg_prefix}API响应了预期的错误状态码 {status_code}. "
f"响应体中的业务错误码 (\'{error_code_field_in_body}\': \'{json_content.get(error_code_field_in_body)}\') 与特定期望 \'{specific_business_error_code}\' 不符或未找到,但HTTP状态码正确。"
))
self.logger.info(f"{self.id}: Passed. HTTP status {status_code} is correct. Business code mismatch/missing (Expected: \'{specific_business_error_code}\', Got: \'{json_content.get(error_code_field_in_body)}\'). (Query param: {context_param_identifier})")
elif business_code_ok:
results.append(self.passed(
f"{context_msg_prefix}API响应了状态码 {status_code} (非主要预期HTTP状态 {expected_http_status_codes},但为4xx客户端错误), "
f"且响应体中包含预期的业务错误码 '{specific_business_error_code}' (字段: '{error_code_field_in_body}')."
))
self.logger.info(f"{self.id}: Passed (Fallback). HTTP status {status_code} (4xx) with matching business code '{specific_business_error_code}'. (Query param: {context_param_identifier})")
else:
fail_message = f"{context_msg_prefix}期望API返回状态码在 {expected_http_status_codes} 中,或返回4xx客户端错误且业务码为 '{specific_business_error_code}'."
fail_message += f" 实际收到状态码 {status_code}."
if json_content and isinstance(json_content, dict):
fail_message += f" 响应体中的业务码 (\'{error_code_field_in_body}\') 为 \'{json_content.get(error_code_field_in_body)}\'."
elif json_content:
fail_message += " 响应体不是一个JSON对象."
else:
fail_message += " 响应体为空或非JSON."
actual_business_code = json_content.get("code")
if str(actual_business_code) != expected_business_error_code:
return [self.failed(
message=f"{context_msg_prefix}业务错误码应为 '{expected_business_error_code}',但实际为 '{actual_business_code}'",
details={"expected_code": expected_business_error_code, "actual_code": actual_business_code, "response_body": json_content}
)]
results.append(self.failed(
message=fail_message,
details={"status_code": status_code, "response_body": json_content, "expected_http_status_codes": expected_http_status_codes, "expected_business_code": specific_business_error_code, "mismatched_param": context_param_identifier}
))
self.logger.warning(f"{self.id}: Failed. {fail_message} (Query param: {context_param_identifier})")
return results
return [self.passed(
f"{context_msg_prefix}API正确返回了状态码 {expected_http_status_code} 和业务错误码 '{expected_business_error_code}'"
)]
def generate_path_params(self, current_path_params: Dict[str, Any]) -> Dict[str, Any]:
# ... existing code ...
+286
View File
@@ -0,0 +1,286 @@
from typing import Dict, Any, Optional, List, Union, Set
import re
import json
import logging
import os
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
from ddms_compliance_suite.utils import schema_utils
class EncryptedFieldsCheck(BaseAPITestCase):
"""
检查API响应中的敏感字段是否已加密。
该测试用例允许用户定义一组需要加密的敏感字段名称,然后检查响应中这些字段是否为明文。
支持多种方式配置敏感字段列表:
1. 使用默认预定义的敏感字段集合
2. 通过API规范中的x-sensitive-fields扩展字段指定
3. 通过环境变量SENSITIVE_FIELDS指定(逗号分隔的字符串)
"""
# 元数据
id = "TC-SECURITY-002"
name = "敏感字段加密检查"
description = "验证API响应中的敏感字段是否已加密,而非明文。"
severity = TestSeverity.HIGH
tags = ["security", "encryption", "sensitive_data"]
execution_order = 50 # 在基本状态码检查之后执行
# 默认需要检查的敏感字段名称集合
DEFAULT_SENSITIVE_FIELDS = {
"coord","location","position"
}
# 加密字符串的正则表达式模式
# 这里假设加密字符串通常是base64或hex编码的字符串
ENCRYPTED_PATTERNS = [
# Base64编码模式 (标准base64字符集,长度通常是4的倍数且较长)
r'^[A-Za-z0-9+/]{20,}={0,2}$',
# Hex编码模式 (16进制字符,长度通常是偶数且较长)
r'^[0-9a-fA-F]{16,}$',
# JWT Token模式 (三段base64编码,用点分隔)
r'^[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+$',
# 通用哈希值模式 (固定长度的hex字符串,如MD5, SHA1, SHA256等)
r'^[0-9a-fA-F]{32}$|^[0-9a-fA-F]{40}$|^[0-9a-fA-F]{64}$',
# 带前缀的加密字符串 (如bcrypt, PBKDF2等)
r'^\$2[ayb]\$.{56}$|^\$pbkdf2-sha\d+\$',
]
# 明文检测模式 (如果匹配这些模式,则可能是明文)
PLAINTEXT_PATTERNS = [
# 常见的明文密码模式 (字母数字特殊字符组合,长度通常在6-20之间)
r'^[A-Za-z0-9!@#$%^&*()_+\-=\[\]{};\':"\\|,.<>\/?]{6,20}$',
# 手机号模式 (中国手机号)
r'^1[3-9]\d{9}$',
# 身份证号模式 (中国身份证号)
r'^\d{17}[\dXx]$|^\d{15}$',
# 银行卡号模式 (数字,通常12-19位)
r'^\d{12,19}$',
# 邮箱模式
r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$',
]
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service)
# 获取敏感字段列表,优先级:API规范扩展字段 > 环境变量 > 默认值
self.sensitive_fields = self._get_sensitive_fields(endpoint_spec)
self.logger.info(f"测试用例 {self.id} ({self.name}) 已初始化,将检查以下敏感字段是否加密: {self.sensitive_fields}")
def _get_sensitive_fields(self, endpoint_spec: Dict[str, Any]) -> Set[str]:
return self.DEFAULT_SENSITIVE_FIELDS
"""
获取敏感字段列表,按以下优先级:
1. API规范中的x-sensitive-fields扩展字段
2. 环境变量SENSITIVE_FIELDS
3. 默认敏感字段列表DEFAULT_SENSITIVE_FIELDS
Args:
endpoint_spec: API端点规范
Returns:
敏感字段集合
"""
custom_fields = set()
# 1. 从API规范中的x-sensitive-fields扩展字段获取
if "x-sensitive-fields" in endpoint_spec:
try:
fields = endpoint_spec["x-sensitive-fields"]
if isinstance(fields, list):
custom_fields.update(fields)
elif isinstance(fields, str):
# 如果是JSON字符串,尝试解析
try:
parsed_fields = json.loads(fields)
if isinstance(parsed_fields, list):
custom_fields.update(parsed_fields)
elif isinstance(parsed_fields, dict) and "fields" in parsed_fields:
# 支持 {"fields": ["field1", "field2"]} 格式
if isinstance(parsed_fields["fields"], list):
custom_fields.update(parsed_fields["fields"])
except json.JSONDecodeError:
# 如果不是JSON,假设是逗号分隔的字符串
custom_fields.update([f.strip() for f in fields.split(",")])
except Exception as e:
self.logger.warning(f"解析API规范中的自定义敏感字段时出错: {e}")
# 2. 如果API规范中没有定义,尝试从环境变量获取
if not custom_fields:
env_fields = os.environ.get("SENSITIVE_FIELDS")
if env_fields:
try:
# 尝试解析为JSON
try:
parsed_fields = json.loads(env_fields)
if isinstance(parsed_fields, list):
custom_fields.update(parsed_fields)
except json.JSONDecodeError:
# 如果不是JSON,假设是逗号分隔的字符串
custom_fields.update([f.strip() for f in env_fields.split(",")])
except Exception as e:
self.logger.warning(f"解析环境变量中的自定义敏感字段时出错: {e}")
# 3. 如果以上方式都没有获取到自定义字段,使用默认值
if not custom_fields:
self.logger.info("未找到自定义敏感字段配置,使用默认敏感字段列表")
return self.DEFAULT_SENSITIVE_FIELDS
else:
self.logger.info(f"使用自定义敏感字段列表: {custom_fields}")
return custom_fields
def is_likely_encrypted(self, value: str) -> bool:
"""
判断一个字符串是否可能是加密的。
Args:
value: 要检查的字符串值
Returns:
如果字符串可能是加密的,则返回True;否则返回False
"""
# 如果值太短,可能不是有效的加密值
if len(value) < 16:
return False
# 检查是否匹配任何加密模式
for pattern in self.ENCRYPTED_PATTERNS:
if re.match(pattern, value):
return True
# 检查是否匹配任何明文模式
for pattern in self.PLAINTEXT_PATTERNS:
if re.match(pattern, value):
return False
# 如果没有匹配任何模式,默认认为是加密的(保守策略)
return True
def find_sensitive_fields_in_response(self, response_data: Any, path: List[Union[str, int]] = None) -> Dict[str, List[Union[str, int]]]:
"""
递归搜索响应数据中的敏感字段。
Args:
response_data: 响应数据(可能是字典、列表或基本类型)
path: 当前路径
Returns:
包含敏感字段路径的字典 {字段名: 路径}
"""
if path is None:
path = []
sensitive_fields_found = {}
self.logger.info(f"response_data:{response_data}")
if isinstance(response_data, dict):
for key, value in response_data.items():
current_path = path + [key]
# 检查当前字段名是否包含任何敏感关键词,不再限制值必须是字符串类型
key_lower = key.lower()
for sensitive_field in self.sensitive_fields:
sensitive_field_lower = sensitive_field.lower()
# 修改为包含匹配而不是严格匹配
if sensitive_field_lower in key_lower or key_lower in sensitive_field_lower:
# 移除对值类型的检查,允许任何类型的敏感字段
sensitive_fields_found[key] = current_path
self.logger.info(f"找到敏感字段: {key}, 匹配关键词: {sensitive_field}")
break
# 递归检查子字段
if isinstance(value, (dict, list)):
nested_fields = self.find_sensitive_fields_in_response(value, current_path)
sensitive_fields_found.update(nested_fields)
elif isinstance(response_data, list):
for i, item in enumerate(response_data):
current_path = path + [i]
if isinstance(item, (dict, list)):
nested_fields = self.find_sensitive_fields_in_response(item, current_path)
sensitive_fields_found.update(nested_fields)
return sensitive_fields_found
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""
验证响应中的敏感字段是否已加密。
Args:
response_context: API响应上下文
request_context: API请求上下文
Returns:
验证结果列表
"""
results = []
# 如果响应不是JSON格式,则跳过检查
if not response_context.json_content:
results.append(self.passed("响应不是JSON格式,跳过敏感字段加密检查。"))
return results
self.logger.info(f"response_context.json_content: {response_context.json_content}")
# 查找响应中的敏感字段
sensitive_fields_found = self.find_sensitive_fields_in_response(response_context.json_content)
if not sensitive_fields_found:
self.logger.info(f"未在响应中找到需要检查的敏感字段。")
results.append(self.passed("未在响应中找到需要检查的敏感字段。"))
return results
# 检查每个敏感字段是否已加密
for field_name, field_path in sensitive_fields_found.items():
# 获取字段值
current_data = response_context.json_content
try:
for path_part in field_path:
current_data = current_data[path_part]
except (KeyError, IndexError, TypeError):
self.logger.warning(f"无法访问路径 {field_path} 的值")
continue
self.logger.info(f"检查敏感字段: {field_name}, 路径: {field_path}, 值类型: {type(current_data)}, 值: {current_data}")
# 根据字段值类型进行不同处理
if isinstance(current_data, str):
# 字符串类型:检查是否已加密
if self.is_likely_encrypted(current_data):
results.append(self.passed(
f"敏感字段 '{'.'.join(map(str, field_path))}' 已正确加密。",
{"field_path": field_path, "field_name": field_name}
))
else:
results.append(self.failed(
f"敏感字段 '{'.'.join(map(str, field_path))}' 可能未加密,存在安全风险。",
{
"field_path": field_path,
"field_name": field_name,
"value_preview": current_data[:10] + "..." if len(current_data) > 10 else current_data
}
))
elif current_data is None:
# 空值:跳过检查
results.append(self.passed(
f"敏感字段 '{'.'.join(map(str, field_path))}' 的值为空,跳过加密检查。",
{"field_path": field_path, "field_name": field_name}
))
else:
# 非字符串类型:警告可能存在安全风险
results.append(self.failed(
f"敏感字段 '{'.'.join(map(str, field_path))}' 的值为非字符串类型 ({type(current_data).__name__}),可能存在安全风险。",
{
"field_path": field_path,
"field_name": field_name,
"field_type": type(current_data).__name__,
"value": str(current_data)
}
))
return results
@staticmethod
def passed(message: str, details: Optional[Dict[str, Any]] = None) -> ValidationResult:
"""创建通过的验证结果"""
return ValidationResult(passed=True, message=message, details=details)
@staticmethod
def failed(message: str, details: Optional[Dict[str, Any]] = None) -> ValidationResult:
"""创建失败的验证结果"""
return ValidationResult(passed=False, message=message, details=details)
+44
View File
@@ -0,0 +1,44 @@
# 设置检查测试用例
本目录包含执行在实际API调用之前的设置检查测试用例。
## 必需请求头Schema验证
`required_headers_check.py` 实现了一个测试用例,用于验证API规范中是否包含所有必需的请求头:
- X-Tenant-ID (也接受 tenant-id 作为变体)
- X-Data-Domain (也接受 data-domain 作为变体)
- Authorization
该测试用例不会发送实际的API请求,只会验证API规范的定义是否符合要求。
### 工作原理
1. 测试用例检查每个API端点的规范定义
2. 验证是否包含所有必需的请求头
3. 验证这些请求头是否被标记为必需 (required="1")
4. 生成详细的验证结果,包括哪些请求头缺失或未标记为必需
### 使用方法
这个测试用例会自动被测试框架发现并应用到所有API端点。由于其`execution_order = 0`设置,它会在其他测试用例之前执行。
如果发现API规范中缺少必需的请求头,测试会失败并提供详细的错误信息,指出哪些请求头缺失或未标记为必需。
### 示例结果
成功情况:
```
✅ 测试通过: 所有必需的请求头都已正确定义
```
失败情况:
```
❌ 测试失败: 缺少必需的请求头 X-Data-Domain
❌ 测试失败: 请求头 tenant-id 存在但未标记为必需
```
### 注意事项
1. 此测试用例接受请求头名称的不同变体(如`X-Tenant-ID``tenant-id`
2. 如果API规范设计时有意不包含某些请求头,可能需要修改测试用例的`required_headers`配置
@@ -1,78 +0,0 @@
# from typing import Dict, Any, Optional, List
# from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
# class BasicAPISanityCheckCase(BaseAPITestCase):
# id = "TC-FRAMEWORK-SANITY-001"
# name = "Basic API Sanity Check"
# description = ("Performs a basic API call with default generated data and expects a generally successful "
# "response (e.g., 200, 201, 204). If a response schema is defined for success, "
# "it also validates the response body against it. "
# "If this test case fails, subsequent test cases for this endpoint may be skipped.")
# severity = TestSeverity.CRITICAL
# tags = ["sanity", "framework-setup"]
# # This flag indicates to the orchestrator that if this test fails,
# # subsequent tests for THIS ENDPOINT should be skipped.
# is_critical_setup_test: bool = True
# execution_order = 1 # Ensures this runs first for an endpoint
# # Expected successful HTTP status codes
# EXPECTED_SUCCESS_STATUS_CODES: List[int] = [200, 201, 202, 204]
# def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
# super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service)
# self.target_success_schema: Optional[Dict[str, Any]] = None
# # Try to find a schema for a successful response (e.g., 200 or 201)
# responses_spec = self.endpoint_spec.get("responses", {})
# if isinstance(responses_spec, dict):
# for status_code_str in map(str, self.EXPECTED_SUCCESS_STATUS_CODES):
# if status_code_str in responses_spec:
# response_def = responses_spec[status_code_str]
# if isinstance(response_def, dict):
# content = response_def.get("content", {})
# for ct in ["application/json", "application/*+json", "*/*"]:
# if ct in content:
# media_type_obj = content[ct]
# if isinstance(media_type_obj, dict) and isinstance(media_type_obj.get("schema"), dict):
# self.target_success_schema = media_type_obj["schema"]
# self.logger.info(f"[{self.id}] Found success response schema for status {status_code_str} under content type {ct}.")
# break # Found a schema for this content type
# if self.target_success_schema:
# break # Found a schema for this status code
# if not self.target_success_schema:
# self.logger.info(f"[{self.id}] No specific success response JSON schema found to validate against for this endpoint.")
# # No need to override generate_* methods, as we want the default behavior.
# def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
# results = []
# status_code = response_context.status_code
# if status_code in self.EXPECTED_SUCCESS_STATUS_CODES:
# msg = f"Basic sanity check: Received expected success status code {status_code}."
# results.append(self.passed(msg))
# # If we have a schema for successful responses, validate the body
# if self.target_success_schema:
# if response_context.json_content is not None:
# results.extend(self.validate_data_against_schema(
# data_to_validate=response_context.json_content,
# schema_definition=self.target_success_schema,
# context_message_prefix="Successful response body"
# ))
# elif response_context.text_content and not response_context.text_content.strip() and status_code == 204:
# # HTTP 204 No Content, body is expected to be empty, so schema validation is not applicable.
# results.append(self.passed("Response is 204 No Content, body is correctly empty."))
# elif status_code != 204 : # For 200, 201, 202, if schema is present, content is expected
# results.append(self.failed(
# message="Basic sanity check: Response body is empty or not JSON, but a success schema was defined.",
# details={"status_code": status_code, "content_type": response_context.headers.get("Content-Type")}
# ))
# else:
# results.append(self.failed(
# message=f"Basic sanity check: Expected a success status code (one of {self.EXPECTED_SUCCESS_STATUS_CODES}), but received {status_code}.",
# details={"status_code": status_code, "response_body": response_context.json_content if response_context.json_content else response_context.text_content}
# ))
# return results
@@ -0,0 +1,127 @@
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, TestSeverity, ValidationResult, APIRequestContext, APIResponseContext
import logging
import json
from typing import Dict, Any, Optional, List
class RequiredHeadersSchemaCheck(BaseAPITestCase):
"""验证API规范中是否包含必需的请求头"""
# 1. 元数据
id = "TC-HEADER-001"
name = "必需请求头Schema验证"
description = "验证API规范中是否包含必需的请求头(X-Tenant-ID、X-Data-Domain和Authorization)"
severity = TestSeverity.CRITICAL
tags = ["headers", "schema", "compliance"]
execution_order = 0 # 优先执行
# is_critical_setup_test = True
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator=json_schema_validator, llm_service=llm_service)
self.logger = logging.getLogger(self.__class__.__name__)
self.logger.info(f"测试用例 {self.id} ({self.name}) 已针对端点 '{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}' 初始化。")
# 定义必需的请求头和可能的命名变体
# self.required_headers = {
# 'X-Tenant-ID': ['X-Tenant-ID', 'tenant-id', 'X-TENANT-ID', 'TENANT-ID'],
# 'X-Data-Domain': ['X-Data-Domain', 'data-domain', 'X-DATA-DOMAIN', 'DATA-DOMAIN'],
# 'Authorization': ['Authorization', 'authorization', 'AUTHORIZATION']
# }
self.required_headers = {
'X-Tenant-ID': ['X-Tenant-ID'],
'X-Data-Domain': ['X-Data-Domain'],
'Authorization': ['Authorization']
}
def pre_request(self, request_context: APIRequestContext) -> APIRequestContext:
"""这个测试用例不需要发送实际请求"""
# 设置一个标志,表示不需要发送请求
self._skip_request = True
return request_context
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
"""验证API规范中是否包含所有必需的请求头"""
results = []
# 从parameters数组中获取header类型的参数
parameters = self.endpoint_spec.get('parameters', [])
header_params = [p for p in parameters if p.get('in') == 'header']
# 记录调试信息
self.logger.info(f"API端点: {self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}")
self.logger.info(f"发现的header参数: {json.dumps(header_params, ensure_ascii=False)}")
# 记录找到的请求头,方便调试
found_headers = {}
# 检查每个必需的请求头
for header_key, possible_names in self.required_headers.items():
header_found = False
required_found = False
found_name = None
# 检查是否存在任何变体的请求头名称
for name in possible_names:
for header in header_params:
header_name = header.get('name', '')
if header_name.lower() == name.lower():
header_found = True
found_name = header_name
# 检查是否被标记为必需
if header.get('required') is True:
required_found = True
break
if header_found:
break
found_headers[header_key] = {
'found': header_found,
'required': required_found,
'name': found_name
}
# 记录检查结果
self.logger.info(f"检查请求头 {header_key}: 找到={header_found}, 必需={required_found}, 名称={found_name}")
# 根据检查结果添加验证结果
if not header_found:
results.append(
ValidationResult(
passed=False,
message=f"缺少必需的请求头 {header_key}",
details={
'header': header_key,
'possible_names': possible_names,
'endpoint': f"{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}"
}
)
)
self.logger.warning(f"规范验证失败: 缺少必需的请求头 {header_key}")
elif not required_found:
results.append(
ValidationResult(
passed=False,
message=f"请求头 {found_name} 存在但未标记为必需",
details={
'header': header_key,
'found_name': found_name,
'endpoint': f"{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}"
}
)
)
self.logger.warning(f"规范验证失败: 请求头 {found_name} 存在但未标记为必需")
# 如果没有失败的验证结果,添加一个成功的结果
if not [r for r in results if not r.passed]:
results.append(
ValidationResult(
passed=True,
message=f"所有必需的请求头都已正确定义",
details={
'found_headers': found_headers,
'endpoint': f"{self.endpoint_spec.get('method')} {self.endpoint_spec.get('path')}"
}
)
)
self.logger.info(f"规范验证通过: 所有必需的请求头都已正确定义")
return results
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,40 @@
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, ValidationResult, APIResponseContext, APIRequestContext, TestSeverity
import re
from typing import Dict, Any, List, Optional
class ResourcePathNounCheckTestCase(BaseAPITestCase):
id = "TC-RESTful-002"
name = "资源路径名词检查"
description = "验证API路径中是否使用名词而非动词来表示资源。"
severity = TestSeverity.MEDIUM
tags = ["normative", "restful", "url-structure"]
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
# 简单的动词列表,可以根据需要扩展
self.common_verbs = {"get", "create", "update", "delete", "post", "put", "add", "remove", "set"}
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
path = self.endpoint_spec['path']
path_segments = [seg for seg in path.split('/') if seg and '{' not in seg]
is_valid = True
offending_verbs = []
for segment in path_segments:
# 移除版本号等非资源路径部分
if re.match(r'v\d+', segment):
continue
# 检查分段是否像动词
# 为了避免误判 (e.g., /dataset),只对完全匹配的动词进行判断
if segment.lower() in self.common_verbs:
is_valid = False
offending_verbs.append(segment)
if not is_valid:
message = f"路径 '{path}' 中可能包含动词: {', '.join(offending_verbs)},RESTful风格建议资源路径使用名词。"
return [self.failed(message, details={'path': path, 'detected_verbs': offending_verbs})]
message = f"路径 '{path}' 符合资源名词命名规范。"
return [self.passed(message)]
@@ -0,0 +1,79 @@
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, ValidationResult, APIResponseContext, APIRequestContext, TestSeverity
import re
from typing import Dict, Any, List, Optional
from ddms_compliance_suite.utils import schema_utils
class TimeFormatCheckTestCase(BaseAPITestCase):
id = "TC-RESTful-003"
name = "时间字段ISO 8601格式检查"
description = "验证返回的时间字段是否遵循 YYYY-MM-DDTHH:MM:SS+08:00 的ISO 8601格式。此检查为静态检查,验证规范中`string`类型且`format`为`date-time`的字段是否包含推荐的`pattern`。"
severity = TestSeverity.MEDIUM
tags = ["normative", "schema", "time-format"]
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
# 推荐的 pattern
self.recommended_iso_8601_pattern = r'^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([+-]\d{2}:\d{2}|Z)$'
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
results = []
# 此检查为静态检查,分析API规范中的响应部分
responses = self.endpoint_spec.get('responses', {})
for status_code, response_spec in responses.items():
# 通常只关心成功的响应
if not status_code.startswith('2'):
continue
content = self._get_resolved_schema(response_spec.get('content', {}))
if content:
for media_type, media_spec in content.items():
if 'schema' in media_spec:
self._check_schema_properties(media_spec['schema'], results)
if not results:
return [self.passed("在API规范中未找到可供静态检查的时间相关字段(类型为string且格式为date-time)。")]
return results
def _check_schema_properties(self, schema, results, path=""):
if not schema or not isinstance(schema, dict):
return
# 处理 allOf, oneOf, anyOf
for keyword in ['allOf', 'oneOf', 'anyOf']:
if keyword in schema:
for sub_schema in schema[keyword]:
self._check_schema_properties(sub_schema, results, path)
if 'properties' in schema:
for prop_name, prop_spec in schema['properties'].items():
prop_spec = self._get_resolved_schema(prop_spec)
current_path = f"{path}.{prop_name}" if path else prop_name
# 检查类型为string且格式为date-time的字段
if prop_spec.get('type') == 'string' and prop_spec.get('format') == 'date-time':
pattern = prop_spec.get('pattern')
message = f"时间字段 '{current_path}' (format: date-time) "
if not pattern:
results.append(self.failed(
message + f"缺少建议的 `pattern` ({self.recommended_iso_8601_pattern}) 来强制执行ISO 8601格式。",
details={'field': current_path}
))
elif pattern != self.recommended_iso_8601_pattern:
results.append(self.failed(
message + f"其 `pattern` ('{pattern}') 与建议的模式不完全匹配。",
details={'field': current_path, 'current_pattern': pattern, 'recommended': self.recommended_iso_8601_pattern}
))
else:
results.append(self.passed(message + "已定义了建议的 `pattern` 用于格式校验。"))
# 递归检查
if 'properties' in prop_spec or 'allOf' in prop_spec or 'oneOf' in prop_spec or 'anyOf' in prop_spec:
self._check_schema_properties(prop_spec, results, current_path)
elif prop_spec.get('type') == 'array' and 'items' in prop_spec:
self._check_schema_properties(self._get_resolved_schema(prop_spec['items']), results, f"{current_path}[]")
def _get_resolved_schema(self, schema_or_ref):
if '$ref' in schema_or_ref:
return schema_utils.util_resolve_ref(schema_or_ref['$ref'], self.global_api_spec)
return schema_or_ref
@@ -0,0 +1,59 @@
# from ddms_compliance_suite.test_framework_core import BaseAPITestCase, ValidationResult, APIResponseContext, APIRequestContext, TestSeverity
# import re
# from typing import Dict, Any, List, Optional
# # TODO 获取资源的时候复数(get方法list)
# class ResourceCollectionPluralCheckTestCase(BaseAPITestCase):
# id = "TC-RESTful-004"
# name = "资源集合复数命名检查"
# description = "验证表示资源集合的路径是否使用复数形式。动词(如push、send等)不需要使用复数形式。"
# severity = TestSeverity.MEDIUM
# tags = ["normative", "restful", "url-structure"]
# def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
# super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
# # 常见的API路径中的动词,这些不需要使用复数形式
# self.common_verbs = {
# "push", "send", "publish", "subscribe", "create", "update", "delete",
# "get", "set", "add", "remove", "search", "query", "find", "calculate",
# "process", "validate", "verify", "check", "analyze", "export", "import",
# "upload", "download", "sync", "login", "logout", "register", "activate",
# "deactivate", "approve", "reject", "cancel", "confirm", "notify"
# }
# # 已知的单数形式名词,即使不以's'结尾也是正确的
# self.known_singulars = {
# "status", "gas", "analysis", "data", "info", "metadata", "media",
# "equipment", "staff", "fish", "sheep", "deer", "series", "species",
# "aircraft", "offspring", "feedback", "content", "news"
# }
# def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
# path = self.endpoint_spec['path']
# method = self.endpoint_spec['method']
# # 这个检查通常适用于返回列表的GET请求,或者创建资源的POST请求
# if method.lower() not in ['get', 'post']:
# return [self.passed(f"跳过检查:{method} 方法,不适用于资源集合复数检查。")]
# path_segments = [seg for seg in path.strip('/').split('/') if '{' not in seg and not re.match(r'v\d+', seg)]
# if not path_segments:
# return [self.passed("跳过检查:路径不含有效分段。")]
# resource_segment = path_segments[-1]
# # 检查是否为动词
# if resource_segment.lower() in self.common_verbs:
# return [self.passed(f"路径 '{path}' 的最后一个路径分段 '{resource_segment}' 是动词,不需要使用复数形式。")]
# # 检查是否为已知的单数形式名词
# if resource_segment.lower() in self.known_singulars:
# return [self.passed(f"路径 '{path}' 的资源名 '{resource_segment}' 是已知的单数形式名词,符合规范。")]
# # 对于其他名词,检查是否使用复数形式
# if not resource_segment.endswith('s'):
# message = f"路径 '{path}' 的最后一个路径分段 '{resource_segment}' 可能不是复数形式,建议对资源集合使用复数命名。"
# return [self.failed(message, details={'path': path, 'segment': resource_segment})]
# message = f"路径 '{path}' 的资源集合命名 '{resource_segment}' 符合复数命名规范。"
# return [self.passed(message)]
@@ -0,0 +1,185 @@
import re
from typing import Dict, Any, List, Optional
from ddms_compliance_suite.test_framework_core import BaseAPITestCase, ValidationResult, APIResponseContext, APIRequestContext, TestSeverity
from ddms_compliance_suite.utils.common_utils import is_camel_case
from ddms_compliance_suite.utils import schema_utils
class CoreNamingStructureTestCase(BaseAPITestCase):
id = "TC-RESTful-001"
name = "核心命名与结构规范检查"
description = "统一验证API的命名与结构是否遵循规范。包括:1)模块名全小写且用中划线连接;2)URL路径参数使用下划线命名法(snake_case);3)查询参数和请求体字段使用小驼峰命名法(camelCase);4)响应中的空数组为[]而非null;5)数组类型数据被包裹在list字段中。"
severity = TestSeverity.HIGH
tags = ["normative", "restful", "structure", "naming-convention"]
def __init__(self, endpoint_spec: Dict[str, Any], global_api_spec: Dict[str, Any], json_schema_validator: Optional[Any] = None, llm_service: Optional[Any] = None):
super().__init__(endpoint_spec, global_api_spec, json_schema_validator, llm_service=llm_service)
def validate_response(self, response_context: APIResponseContext, request_context: APIRequestContext) -> List[ValidationResult]:
results = []
# 静态检查,只分析API规范
self._check_module_name(results)
self._check_request_params_camel_case(results)
# 动态检查,需要分析实际响应
# 规则 "响应中的空数组为[]而非null" 和 "数组类型数据被包裹在list字段中" 需要在实际调用后进行
# 这里我们只对成功的响应进行检查
if 200 <= response_context.status_code < 300:
self._check_response_array_format(response_context, results)
else:
results.append(self.passed("跳过响应体检查:非成功状态码,不适用数组格式检查。"))
return results
def _check_module_name(self, results: List[ValidationResult]):
"""检查模块名是否为全小写且用中划线连接"""
path = self.endpoint_spec['path']
module_name_match = re.search(r'/api/([^/]+)/', path)
if module_name_match:
module_name = module_name_match.group(1)
# 模块名可以只是小写字母,但如果包含连接符,必须是中划线
is_valid = all(c.islower() or c.isdigit() or c == '-' for c in module_name) and '_' not in module_name
if is_valid:
results.append(self.passed(f"模块名 '{module_name}' 格式正确 (全小写/数字/中划线)。"))
else:
results.append(self.failed(f"模块名 '{module_name}' 格式不正确。应为全小写字母、数字和中划线的组合。",
details={'path': path, 'module': module_name}))
else:
results.append(self.failed(f"无法从路径 '{path}' 中提取模块名(格式应为 /api/module-name/...)。",
details={'path': path}))
def _check_request_params_camel_case(self, results: List[ValidationResult]):
"""检查请求参数命名规范:
- URL路径参数应使用下划线命名法(snake_case)
- 请求体和查询参数应使用小驼峰命名法(camelCase)
- HTTP头部有特殊规范,不检查
"""
parameters = self.endpoint_spec.get('parameters', [])
# 定义HTTP头部例外(不检查)
header_exceptions = ['Authorization', 'X-Tenant-ID', 'X-Data-Domain', 'tenant-id', 'Content-Type']
for param in parameters:
param_name = param.get('name')
param_in = param.get('in')
# 跳过HTTP头部参数
if param_in == 'header' and param_name in header_exceptions:
continue
# 路径参数使用下划线命名法(snake_case)
if param_in == 'path':
# 下划线命名法:全小写字母、数字和下划线,不允许连续下划线,不能以下划线开头或结尾
is_valid_snake_case = re.match(r'^[a-z][a-z0-9_]*$', param_name) is not None and '__' not in param_name and not param_name.endswith('_')
if not is_valid_snake_case:
results.append(self.failed(f"路径参数 '{param_name}' 不符合下划线命名法(snake_case)规范。应为小写字母、数字和单下划线组合,不能以下划线结尾。",
details={'parameter': param_name, 'location': param_in}))
# 查询参数使用小驼峰命名法(camelCase)
elif param_in == 'query':
if not is_camel_case(param_name):
results.append(self.failed(f"查询参数 '{param_name}' 不是小驼峰格式。",
details={'parameter': param_name, 'location': param_in}))
# 检查请求体
body_schema = self._get_resolved_request_body_schema()
if body_schema:
self._check_schema_properties_camel_case(body_schema, results)
def _check_schema_properties_camel_case(self, schema, results, path=""):
if not schema or not isinstance(schema, dict):
return
if 'properties' in schema:
for prop_name, prop_spec in schema['properties'].items():
if not is_camel_case(prop_name):
full_path = f"{path}.{prop_name}" if path else prop_name
results.append(self.failed(f"请求体字段 '{full_path}' 不是小驼峰格式。",
details={'field': full_path}))
prop_spec_resolved = self._get_resolved_schema(prop_spec)
if 'properties' in prop_spec_resolved or 'items' in prop_spec_resolved:
self._check_schema_properties_camel_case(prop_spec_resolved, results, f"{path}.{prop_name}" if path else prop_name)
def _check_response_array_format(self, response_context: APIResponseContext, results: List[ValidationResult]):
"""检查响应中的空数组和数组包裹"""
json_content = response_context.json_content
if json_content is None:
# 如果响应体为空,则跳过检查
results.append(self.passed("响应体为空,跳过数组格式检查。"))
return
# 检查 "数组类型数据被包裹在list字段中"
# 这条规则比较模糊,这里理解为:如果响应体是一个以数组为核心的列表,那么这个数组的key应该是'list'
if isinstance(json_content, dict) and len(json_content) > 0:
list_keys = [k for k, v in json_content.items() if isinstance(v, list)]
if len(list_keys) == 1 and list_keys[0] != 'list':
results.append(self.failed(f"响应中包含一个主列表,但其键名 '{list_keys[0]}' 不是 'list'",
details={'keys': list(json_content.keys())}))
elif len(list_keys) > 1 and 'list' not in list_keys:
results.append(self.failed(f"响应中包含多个列表,但没有一个的键名是 'list'",
details={'keys': list(json_content.keys())}))
# 检查 "响应中的空数组为[]而非null"
# 查找匹配的响应定义时,需要兼容 status_code, status_code_family (e.g., 2XX), 和 default
responses = self.endpoint_spec.get('responses', {})
self.logger.info(f"responses: {responses}")
status_code = response_context.status_code
print("status_code: ", status_code)
status_code_str = str(status_code)
status_code_family = f"{status_code_str[0]}XX" # e.g., "2XX"
self.logger.info(f"检查响应定义: status_code={status_code}, 可用响应定义={list(responses.keys())}")
# 按优先级查找响应定义:精确状态码 > 状态码族 > 默认状态码200 > default
response_spec = None
print("responses: ", responses)
if status_code_str in responses:
response_spec = responses[status_code_str]
self.logger.info(f"找到精确状态码 {status_code_str} 的响应定义")
elif status_code_family in responses:
response_spec = responses[status_code_family]
self.logger.info(f"找到状态码族 {status_code_family} 的响应定义")
elif '200' in responses and 200 <= status_code < 300: # 对于2XX成功响应,尝试使用200定义
response_spec = responses['200']
self.logger.info(f"未找到状态码 {status_code_str} 的响应定义,使用默认成功状态码200的定义")
elif 'default' in responses:
response_spec = responses['default']
self.logger.info(f"使用default响应定义")
if not response_spec:
results.append(self.passed(f"规范中未找到响应码 {status_code} 或其类别({status_code_family}, default)的匹配定义,跳过空数组与null的检查。"))
return
# 使用基类中定义好的工具函数获取响应schema
schema = self._get_resolved_response_schema(response_spec=response_spec)
if not schema:
results.append(self.passed(f"规范中响应码 {status_code} 的定义中未找到Schema,跳过空数组与null的检查。"))
return
self._validate_null_for_array(json_content, schema, results, "")
def _validate_null_for_array(self, data: Any, schema: Dict[str, Any], results: List[ValidationResult], path: str):
if not schema:
return
schema = self._get_resolved_schema(schema)
if schema.get('type') == 'array' and data is None:
results.append(self.failed(f"响应中字段 '{path}' 的值为 null,但其在规范中定义为数组,应返回 []。", details={'field': path}))
return
if isinstance(data, dict) and 'properties' in schema:
for prop_name, prop_schema in schema['properties'].items():
if prop_name in data:
new_path = f"{path}.{prop_name}" if path else prop_name
self._validate_null_for_array(data[prop_name], prop_schema, results, new_path)
elif isinstance(data, list) and 'items' in schema:
item_schema = schema.get('items')
for i, item in enumerate(data):
new_path = f"{path}[{i}]"
self._validate_null_for_array(item, item_schema, results, new_path)