History

feat(sdk): check crawl/batch scrape errors

Gergő Móricz committed 1 year ago

Commit 146dc471 parent dbc6d072 files · +105 −0Browse files

Showing the changes to apps/python-sdk (1 of 2 files). Show all

Unified · Split

apps/python-sdk/firecrawl/firecrawl.py+39−0
@@ -309,6 +309,26 @@ class FirecrawlApp: · 308 unmodified lines
309 else:309 else:
310 self._handle_error(response, 'check crawl status')310 self._handle_error(response, 'check crawl status')
311 311
312 def check_crawl_errors(self, id: str) -> Dict[str, Any]:
313 """
314 Returns information about crawl errors.
315
316 Args:
317 id (str): The ID of the crawl job.
318
319 Returns:
320 Dict[str, Any]: Information about crawl errors.
321 """
322 headers = self._prepare_headers()
323 response = self._get_request(f'{self.api_url}/v1/crawl/{id}/errors', headers)
324 if response.status_code == 200:
325 try:
326 return response.json()
327 except:
328 raise Exception(f'Failed to parse Firecrawl response as JSON.')
329 else:
330 self._handle_error(response, "check crawl errors")
331
312 def cancel_crawl(self, id: str) -> Dict[str, Any]:332 def cancel_crawl(self, id: str) -> Dict[str, Any]:
313 """333 """
314 Cancel an asynchronous crawl job using the Firecrawl API.334 Cancel an asynchronous crawl job using the Firecrawl API.
@@ -546,6 +566,25 @@ class FirecrawlApp: · 231 unmodified lines
546 else:566 else:
547 self._handle_error(response, 'check batch scrape status')567 self._handle_error(response, 'check batch scrape status')
548 568
569 def check_batch_scrape_errors(self, id: str) -> Dict[str, Any]:
570 """
571 Returns information about batch scrape errors.
572
573 Args:
574 id (str): The ID of the crawl job.
575
576 Returns:
577 Dict[str, Any]: Information about crawl errors.
578 """
579 headers = self._prepare_headers()
580 response = self._get_request(f'{self.api_url}/v1/batch/scrape/{id}/errors', headers)
581 if response.status_code == 200:
582 try:
583 return response.json()
584 except:
585 raise Exception(f'Failed to parse Firecrawl response as JSON.')
586 else:
587 self._handle_error(response, "check batch scrape errors")
549 588
550 def extract(self, urls: List[str], params: Optional[ExtractParams] = None) -> Any:589 def extract(self, urls: List[str], params: Optional[ExtractParams] = None) -> Any:
551 """590 """