add:page

2025-08-18 23:56:08 +08:00
parent 2ae75b8c75
commit 336913fbd0
8 changed files with 1086 additions and 54 deletions
@@ -538,7 +538,7 @@ class InputParser:
            self.logger.error(f"An unexpected error occurred while parsing Swagger spec {file_path}: {e}", exc_info=True)
        return None

-    def parse_dms_spec(self, domain_mapping_path: str, base_url: str, headers: Optional[Dict[str, str]] = None, ignore_ssl: bool = False) -> Optional[ParsedDMSSpec]:
+    def parse_dms_spec(self, domain_mapping_path: str, base_url: str, headers: Optional[Dict[str, str]] = None, ignore_ssl: bool = False, page_size: int = 1000) -> Optional[Tuple[ParsedDMSSpec, Dict[str, Any]]]:
        self.logger.info(f"Starting DMS spec parsing. Base URL: {base_url}, Domain Map: {domain_mapping_path}")

        if ignore_ssl:
@@ -565,31 +565,75 @@ class InputParser:
                        keyword_to_domain_id[keyword] = domain_id
                        self.logger.debug(f"映射关键词 '{keyword}' -> 领域ID '{domain_id}'")
    
-        list_url = urljoin(base_url, "/api/schema/manage/schema?pageNo=1&pageSize=100000")
-        self.logger.info(f"Fetching API list from: {list_url}")
+        # 实现分页获取API列表
+        self.logger.info(f"Fetching API list with pagination (page_size={page_size})")
+        api_records = []
+        page_no = 1
+        total_fetched = 0
+        pagination_info = {
+            "page_size": page_size,
+            "total_pages": 0,
+            "total_records": 0,
+            "pages_fetched": 0
+        }
+
        try:
-            response = requests.get(list_url, headers=headers, verify=not ignore_ssl)
-            response.raise_for_status()
-            api_list_data = response.json()
+            while True:
+                list_url = urljoin(base_url, f"/api/schema/manage/schema?pageNo={page_no}&pageSize={page_size}")
+                self.logger.debug(f"Fetching page {page_no} from: {list_url}")
+
+                response = requests.get(list_url, headers=headers, verify=not ignore_ssl)
+                response.raise_for_status()
+                api_list_data = response.json()
+
+                # 检查业务代码是否成功
+                if api_list_data.get("code") != 0:
+                    self.logger.error(f"DMS API list endpoint returned a business error: {api_list_data.get('message')}")
+                    return None, {}
+
+                # 从分页结构中提取 'records'
+                page_records = api_list_data.get("data", {}).get("records", [])
+                if not page_records:
+                    self.logger.info(f"No more records found on page {page_no}, stopping pagination")
+                    break
+
+                api_records.extend(page_records)
+                total_fetched += len(page_records)
+                self.logger.info(f"Fetched {len(page_records)} records from page {page_no}, total: {total_fetched}")
+
+                # 更新分页信息
+                data = api_list_data.get("data", {})
+                total_count = data.get("total", 0)
+                current_count = data.get("current", 0) * data.get("size", page_size)
+
+                # 第一次获取时更新总数信息
+                if page_no == 1:
+                    pagination_info["total_records"] = total_count
+                    pagination_info["total_pages"] = (total_count + page_size - 1) // page_size  # 向上取整
+
+                pagination_info["pages_fetched"] = page_no
+
+                if current_count >= total_count or len(page_records) < page_size:
+                    self.logger.info(f"Reached end of data. Total records: {total_fetched}")
+                    break
+
+                page_no += 1
+
+                # 安全检查：防止无限循环
+                if page_no > 1000:  # 最多1000页
+                    self.logger.warning("Reached maximum page limit (1000), stopping pagination")
+                    break

-            # 检查业务代码是否成功
-            if api_list_data.get("code") != 0:
-                self.logger.error(f"DMS API list endpoint returned a business error: {api_list_data.get('message')}")
-                return None
-            
-            # 从分页结构中提取 'records'
-            api_records = api_list_data.get("data", {}).get("records", [])
            if not api_records:
-                self.logger.warning("DMS API list is empty or 'records' key is missing in the response data.")
-                # Returning an empty spec is valid if the list is just empty.
-                return ParsedDMSSpec(endpoints=[], spec={"dms_api_list": []})
+                self.logger.warning("DMS API list is empty after pagination.")
+                return ParsedDMSSpec(endpoints=[], spec={"dms_api_list": []}), pagination_info

        except requests.exceptions.RequestException as e:
            self.logger.error(f"Failed to fetch API list from DMS: {e}")
-            return None
+            return None, {}
        except json.JSONDecodeError:
            self.logger.error("Failed to decode JSON response from DMS API list.")
-            return None
+            return None, {}

        endpoints: List[DMSEndpoint] = []
        
@@ -756,7 +800,7 @@ class InputParser:
        # The 'spec' for ParsedDMSSpec should represent the whole document.
        # We can construct a dictionary holding all the raw data we fetched.
        dms_full_spec_dict = {"dms_api_list": api_records}
-        return ParsedDMSSpec(endpoints=endpoints, spec=dms_full_spec_dict)
+        return ParsedDMSSpec(endpoints=endpoints, spec=dms_full_spec_dict), pagination_info

 class DmsConfig:
    def __init__(self, base_url: str, domain_map_file: str, headers: Optional[Dict[str, str]] = None):
@@ -2681,8 +2681,9 @@ class APITestOrchestrator:
    def run_tests_from_dms(self, domain_mapping_path: str,
                           categories: Optional[List[str]] = None,
                           custom_test_cases_dir: Optional[str] = None,
-                           ignore_ssl: bool = False
-                           ) -> Tuple[TestSummary, Optional[ParsedAPISpec]]:
+                           ignore_ssl: bool = False,
+                           page_size: int = 1000
+                           ) -> Tuple[TestSummary, Optional[ParsedAPISpec], Dict[str, Any]]:
        """
        通过动态DMS服务发现来执行测试。
        """
@@ -2692,20 +2693,22 @@ class APITestOrchestrator:
        self.logger.info("从DMS动态服务启动测试...")
        # 如果方法参数中没有传递ignore_ssl，使用实例的设置
        actual_ignore_ssl = ignore_ssl if ignore_ssl else self.ignore_ssl
-        parsed_spec = parser.parse_dms_spec(domain_mapping_path, base_url=self.base_url, ignore_ssl=actual_ignore_ssl)
+        parse_result = parser.parse_dms_spec(domain_mapping_path, base_url=self.base_url, ignore_ssl=actual_ignore_ssl, page_size=page_size)

-        if not parsed_spec:
+        if not parse_result or parse_result[0] is None:
            self.logger.error("无法从DMS服务解析API，测试终止。")
            summary.add_error("Could not parse APIs from DMS service.")
            summary.finalize_summary()
-            return summary, None
+            return summary, None, {}
+
+        parsed_spec, pagination_info = parse_result

        # 🔧 移除重复的run_stages_from_spec调用
        # Stage执行将在主程序中统一处理

        summary = self._execute_tests_from_parsed_spec(parsed_spec, summary, categories=categories, custom_test_cases_dir=custom_test_cases_dir)
        summary.finalize_summary()
-        return summary, parsed_spec
+        return summary, parsed_spec, pagination_info

    def _validate_status_code(self, actual_code: int, expected_codes: List[int]) -> ValidationResult:
        """Helper to validate the HTTP status code."""