"""Example demonstrating pagination functionality in Firecrawl v2 SDK. This example shows how to use the new pagination features for both crawl and batch scrape operations.""" from firecrawl import Firecrawlfrom firecrawl.v2.types import PaginationConfig # Initialize the clientfirecrawl = Firecrawl(api_key="your-api-key") # Example 1: Simple crawl - automatically waits for completion and returns all results# Use this when you want all the data and don't need to control paginationprint("=== Example 1: Simple crawl (default) ===")crawl_result = firecrawl.crawl("https://example.com", limit=100)print(f"Total documents fetched: {len(crawl_result.data)}")print(f"Next URL: {crawl_result.next}")  # Should be None since auto-pagination is enabled # Example 2: Manual crawl with pagination control# Use this when you need to control how many pages to fetch or want to process results incrementallyprint("\n=== Example 2: Manual crawl with pagination control ===")crawl_job = firecrawl.start_crawl("https://example.com", limit=100) # Get just the first page of results (useful for large crawls where you want to process incrementally)pagination_config = PaginationConfig(auto_paginate=False)status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)print(f"Documents from first page: {len(status.data)}")print(f"Next URL: {status.next}")  # Will show the next page URL # Example 3: Limited pagination - fetch only 3 pages# Useful for controlling memory usage or processing timeprint("\n=== Example 3: Limited pagination ===")pagination_config = PaginationConfig(max_pages=3)status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)print(f"Documents from first 3 pages: {len(status.data)}") # Example 4: Result-limited pagination - stop after 50 results# Useful when you only need a specific number of resultsprint("\n=== Example 4: Result-limited pagination ===")pagination_config = PaginationConfig(max_results=50)status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)print(f"Documents (max 50): {len(status.data)}") # Example 5: Time-limited pagination - stop after 30 seconds# Useful for controlling processing time in time-sensitive applicationsprint("\n=== Example 5: Time-limited pagination ===")pagination_config = PaginationConfig(max_wait_time=30)status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)print(f"Documents fetched within 30 seconds: {len(status.data)}") # Example 6: Combined pagination limits# Useful for fine-grained control over resource usageprint("\n=== Example 6: Combined limits ===")pagination_config = PaginationConfig(    max_pages=5,        # Only fetch 5 pages maximum    max_results=100,    # Only fetch 100 results maximum    max_wait_time=60    # Spend at most 60 seconds fetching additional pages)status = firecrawl.get_crawl_status(crawl_job.id, pagination_config=pagination_config)print(f"Documents with combined limits: {len(status.data)}") # Example 7: Simple batch scrape - automatically waits for completion and returns all results# Use this when you want all the data from multiple URLs and don't need to control paginationprint("\n=== Example 7: Simple batch scrape (default) ===")urls = ["https://example1.com", "https://example2.com", "https://example3.com"]batch_result = firecrawl.batch_scrape(urls)print(f"Batch scrape documents: {len(batch_result.data)}") # Example 8: Manual batch scrape with pagination control# Use this when you need to control how many pages to fetch or want to process results incrementallyprint("\n=== Example 8: Manual batch scrape with pagination control ===")batch_job = firecrawl.start_batch_scrape(urls)status = firecrawl.get_batch_scrape_status(batch_job.id)print(f"Batch scrape documents: {len(status.data)}") # Example 9: Async usageprint("\n=== Example 9: Async pagination ===")import asynciofrom firecrawl import AsyncFirecrawl async def async_example():    async_client = AsyncFirecrawl(api_key="your-api-key")        # Simple async crawl - automatically waits for completion    crawl_result = await async_client.crawl("https://example.com", limit=50)    print(f"Async crawl documents: {len(crawl_result.data)}")        # Manual async crawl with pagination    crawl_job = await async_client.start_crawl("https://example.com", limit=50)    pagination_config = PaginationConfig(max_pages=2)    status = await async_client.get_crawl_status(        crawl_job.id,         pagination_config=pagination_config    )    print(f"Async crawl with pagination: {len(status.data)}") # Run async example# asyncio.run(async_example())