add:page

ddms_compliance_suite/input_parser/parser.py

@@ -538,7 +538,7 @@ class InputParser:
             self.logger.error(f"An unexpected error occurred while parsing Swagger spec {file_path}: {e}", exc_info=True)
         return None
 
-    def parse_dms_spec(self, domain_mapping_path: str, base_url: str, headers: Optional[Dict[str, str]] = None, ignore_ssl: bool = False) -> Optional[ParsedDMSSpec]:
+    def parse_dms_spec(self, domain_mapping_path: str, base_url: str, headers: Optional[Dict[str, str]] = None, ignore_ssl: bool = False, page_size: int = 1000) -> Optional[Tuple[ParsedDMSSpec, Dict[str, Any]]]:
         self.logger.info(f"Starting DMS spec parsing. Base URL: {base_url}, Domain Map: {domain_mapping_path}")
 
         if ignore_ssl:
@@ -565,31 +565,75 @@ class InputParser:
                         keyword_to_domain_id[keyword] = domain_id
                         self.logger.debug(f"映射关键词 '{keyword}' -> 领域ID '{domain_id}'")
     
-        list_url = urljoin(base_url, "/api/schema/manage/schema?pageNo=1&pageSize=100000")
-        self.logger.info(f"Fetching API list from: {list_url}")
+        # 实现分页获取API列表
+        self.logger.info(f"Fetching API list with pagination (page_size={page_size})")
+        api_records = []
+        page_no = 1
+        total_fetched = 0
+        pagination_info = {
+            "page_size": page_size,
+            "total_pages": 0,
+            "total_records": 0,
+            "pages_fetched": 0
+        }
+
         try:
-            response = requests.get(list_url, headers=headers, verify=not ignore_ssl)
-            response.raise_for_status()
-            api_list_data = response.json()
+            while True:
+                list_url = urljoin(base_url, f"/api/schema/manage/schema?pageNo={page_no}&pageSize={page_size}")
+                self.logger.debug(f"Fetching page {page_no} from: {list_url}")
+
+                response = requests.get(list_url, headers=headers, verify=not ignore_ssl)
+                response.raise_for_status()
+                api_list_data = response.json()
+
+                # 检查业务代码是否成功
+                if api_list_data.get("code") != 0:
+                    self.logger.error(f"DMS API list endpoint returned a business error: {api_list_data.get('message')}")
+                    return None, {}
+
+                # 从分页结构中提取 'records'
+                page_records = api_list_data.get("data", {}).get("records", [])
+                if not page_records:
+                    self.logger.info(f"No more records found on page {page_no}, stopping pagination")
+                    break
+
+                api_records.extend(page_records)
+                total_fetched += len(page_records)
+                self.logger.info(f"Fetched {len(page_records)} records from page {page_no}, total: {total_fetched}")
+
+                # 更新分页信息
+                data = api_list_data.get("data", {})
+                total_count = data.get("total", 0)
+                current_count = data.get("current", 0) * data.get("size", page_size)
+
+                # 第一次获取时更新总数信息
+                if page_no == 1:
+                    pagination_info["total_records"] = total_count
+                    pagination_info["total_pages"] = (total_count + page_size - 1) // page_size  # 向上取整
+
+                pagination_info["pages_fetched"] = page_no
+
+                if current_count >= total_count or len(page_records) < page_size:
+                    self.logger.info(f"Reached end of data. Total records: {total_fetched}")
+                    break
+
+                page_no += 1
+
+                # 安全检查：防止无限循环
+                if page_no > 1000:  # 最多1000页
+                    self.logger.warning("Reached maximum page limit (1000), stopping pagination")
+                    break
 
-            # 检查业务代码是否成功
-            if api_list_data.get("code") != 0:
-                self.logger.error(f"DMS API list endpoint returned a business error: {api_list_data.get('message')}")
-                return None
-            
-            # 从分页结构中提取 'records'
-            api_records = api_list_data.get("data", {}).get("records", [])
             if not api_records:
-                self.logger.warning("DMS API list is empty or 'records' key is missing in the response data.")
-                # Returning an empty spec is valid if the list is just empty.
-                return ParsedDMSSpec(endpoints=[], spec={"dms_api_list": []})
+                self.logger.warning("DMS API list is empty after pagination.")
+                return ParsedDMSSpec(endpoints=[], spec={"dms_api_list": []}), pagination_info
 
         except requests.exceptions.RequestException as e:
             self.logger.error(f"Failed to fetch API list from DMS: {e}")
-            return None
+            return None, {}
         except json.JSONDecodeError:
             self.logger.error("Failed to decode JSON response from DMS API list.")
-            return None
+            return None, {}
 
         endpoints: List[DMSEndpoint] = []
         
@@ -756,7 +800,7 @@ class InputParser:
         # The 'spec' for ParsedDMSSpec should represent the whole document.
         # We can construct a dictionary holding all the raw data we fetched.
         dms_full_spec_dict = {"dms_api_list": api_records}
-        return ParsedDMSSpec(endpoints=endpoints, spec=dms_full_spec_dict)
+        return ParsedDMSSpec(endpoints=endpoints, spec=dms_full_spec_dict), pagination_info
 
 class DmsConfig:
     def __init__(self, base_url: str, domain_map_file: str, headers: Optional[Dict[str, str]] = None):