Showing the changes to apps/python-sdk (14 of 31 files). Show all
Unified · Split
| @@ -1,2 +1,5 @@ | |||
| 1 | FIRECRAWL_API_KEY=<your api key> | 1 | FIRECRAWL_API_KEY=<your api key> |
| 2 | FIRECRAWL_API_URL=https://api.firecrawl.dev | 2 | FIRECRAWL_API_URL=https://api.firecrawl.dev |
| 3 | |||
| 4 | # TESTING PURPOSE: | ||
| 5 | IDMUX_URL= | ||
| @@ -13,24 +13,25 @@ pip install firecrawl-py · 12 unmodified lines | |||
| 13 | ## Usage | 13 | ## Usage |
| 14 | 14 | ||
| 15 | 1. Get an API key from [firecrawl.dev](https://firecrawl.dev) | 15 | 1. Get an API key from [firecrawl.dev](https://firecrawl.dev) |
| 16 | 2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class. | 16 | 2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `Firecrawl` class. |
| 17 | 17 | ||
| 18 | Here's an example of how to use the SDK: | 18 | Here's an example of how to use the SDK: |
| 19 | 19 | ||
| 20 | ```python | 20 | ```python |
| 21 | from firecrawl import FirecrawlApp, ScrapeOptions | 21 | from firecrawl import Firecrawl |
| 22 | from firecrawl.types import ScrapeOptions | ||
| 22 | 23 | ||
| 23 | app = FirecrawlApp(api_key="fc-YOUR_API_KEY") | 24 | firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY") |
| 24 | 25 | ||
| 25 | # Scrape a website: | 26 | # Scrape a website (v2): |
| 26 | data = app.scrape_url( | 27 | data = firecrawl.scrape( |
| 27 | 'https://firecrawl.dev', | 28 | 'https://firecrawl.dev', |
| 28 | formats=['markdown', 'html'] | 29 | formats=['markdown', 'html'] |
| 29 | ) | 30 | ) |
| 30 | print(data) | 31 | print(data) |
| 31 | 32 | ||
| 32 | # Crawl a website: | 33 | # Crawl a website (v2 waiter): |
| 33 | crawl_status = app.crawl_url( | 34 | crawl_status = firecrawl.crawl( |
| 34 | 'https://firecrawl.dev', | 35 | 'https://firecrawl.dev', |
| 35 | limit=100, | 36 | limit=100, |
| 36 | scrape_options=ScrapeOptions(formats=['markdown', 'html']) | 37 | scrape_options=ScrapeOptions(formats=['markdown', 'html']) |
| @@ -40,20 +41,20 @@ print(crawl_status) · 3 unmodified lines | |||
| 40 | 41 | ||
| 41 | ### Scraping a URL | 42 | ### Scraping a URL |
| 42 | 43 | ||
| 43 | To scrape a single URL, use the `scrape_url` method. It takes the URL as a parameter and returns the scraped data as a dictionary. | 44 | To scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats. |
| 44 | 45 | ||
| 45 | ```python | 46 | ```python |
| 46 | # Scrape a website: | 47 | # Scrape a website (v2): |
| 47 | scrape_result = app.scrape_url('firecrawl.dev', formats=['markdown', 'html']) | 48 | scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', 'html']) |
| 48 | print(scrape_result) | 49 | print(scrape_result) |
| 49 | ``` | 50 | ``` |
| 50 | 51 | ||
| 51 | ### Crawling a Website | 52 | ### Crawling a Website |
| 52 | 53 | ||
| 53 | To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. | 54 | To crawl a website, use the `crawl` method. It takes the starting URL and optional parameters as arguments. You can control depth, limits, formats, and more. |
| 54 | 55 | ||
| 55 | ```python | 56 | ```python |
| 56 | crawl_status = app.crawl_url( | 57 | crawl_status = firecrawl.crawl( |
| 57 | 'https://firecrawl.dev', | 58 | 'https://firecrawl.dev', |
| 58 | limit=100, | 59 | limit=100, |
| 59 | scrape_options=ScrapeOptions(formats=['markdown', 'html']), | 60 | scrape_options=ScrapeOptions(formats=['markdown', 'html']), |
| @@ -66,23 +67,23 @@ print(crawl_status) · 6 unmodified lines | |||
| 66 | 67 | ||
| 67 | <Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip> | 68 | <Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip> |
| 68 | 69 | ||
| 69 | To crawl a website asynchronously, use the `crawl_url_async` method. It returns the crawl `ID` which you can use to check the status of the crawl job. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format. | 70 | To enqueue a crawl asynchronously, use `start_crawl`. It returns the crawl `ID` which you can use to check the status of the crawl job. |
| 70 | 71 | ||
| 71 | ```python | 72 | ```python |
| 72 | crawl_status = app.async_crawl_url( | 73 | crawl_job = firecrawl.start_crawl( |
| 73 | 'https://firecrawl.dev', | 74 | 'https://firecrawl.dev', |
| 74 | limit=100, | 75 | limit=100, |
| 75 | scrape_options=ScrapeOptions(formats=['markdown', 'html']), | 76 | scrape_options=ScrapeOptions(formats=['markdown', 'html']), |
| 76 | ) | 77 | ) |
| 77 | print(crawl_status) | 78 | print(crawl_job) |
| 78 | ``` | 79 | ``` |
| 79 | 80 | ||
| 80 | ### Checking Crawl Status | 81 | ### Checking Crawl Status |
| 81 | 82 | ||
| 82 | To check the status of a crawl job, use the `check_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job. | 83 | To check the status of a crawl job, use the `get_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job. |
| 83 | 84 | ||
| 84 | ```python | 85 | ```python |
| 85 | crawl_status = app.check_crawl_status("<crawl_id>") | 86 | crawl_status = firecrawl.get_crawl_status("<crawl_id>") |
| 86 | print(crawl_status) | 87 | print(crawl_status) |
| 87 | ``` | 88 | ``` |
| 88 | 89 | ||
| @@ -91,17 +92,17 @@ print(crawl_status) · 2 unmodified lines | |||
| 91 | To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status. | 92 | To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status. |
| 92 | 93 | ||
| 93 | ```python | 94 | ```python |
| 94 | cancel_crawl = app.cancel_crawl(id) | 95 | cancel_crawl = firecrawl.cancel_crawl(id) |
| 95 | print(cancel_crawl) | 96 | print(cancel_crawl) |
| 96 | ``` | 97 | ``` |
| 97 | 98 | ||
| 98 | ### Map a Website | 99 | ### Map a Website |
| 99 | 100 | ||
| 100 | Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap. | 101 | Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to use the sitemap or include subdomains. |
| 101 | 102 | ||
| 102 | ```python | 103 | ```python |
| 103 | # Map a website: | 104 | # Map a website (v2): |
| 104 | map_result = app.map_url('https://firecrawl.dev') | 105 | map_result = firecrawl.map('https://firecrawl.dev') |
| 105 | print(map_result) | 106 | print(map_result) |
| 106 | ``` | 107 | ``` |
| 107 | 108 | ||
| @@ -152,20 +153,35 @@ The SDK handles errors returned by the Firecrawl API and raises appropriate exce · 44 unmodified lines | |||
| 152 | 153 | ||
| 153 | ## Async Class | 154 | ## Async Class |
| 154 | 155 | ||
| 155 | For async operations, you can use the `AsyncFirecrawlApp` class. Its methods are the same as the `FirecrawlApp` class, but they don't block the main thread. | 156 | For async operations, you can use the `AsyncFirecrawl` class. Its methods mirror the `Firecrawl` class, but you `await` them. |
| 156 | 157 | ||
| 157 | ```python | 158 | ```python |
| 158 | from firecrawl import AsyncFirecrawlApp | 159 | from firecrawl import AsyncFirecrawl |
| 159 | 160 | ||
| 160 | app = AsyncFirecrawlApp(api_key="YOUR_API_KEY") | 161 | firecrawl = AsyncFirecrawl(api_key="YOUR_API_KEY") |
| 161 | 162 | ||
| 162 | # Async Scrape | 163 | # Async Scrape (v2) |
| 163 | async def example_scrape(): | 164 | async def example_scrape(): |
| 164 | scrape_result = await app.scrape_url(url="https://example.com") | 165 | scrape_result = await firecrawl.scrape(url="https://example.com") |
| 165 | print(scrape_result) | 166 | print(scrape_result) |
| 166 | 167 | ||
| 167 | # Async Crawl | 168 | # Async Crawl (v2) |
| 168 | async def example_crawl(): | 169 | async def example_crawl(): |
| 169 | crawl_result = await app.crawl_url(url="https://example.com") | 170 | crawl_result = await firecrawl.crawl(url="https://example.com") |
| 170 | print(crawl_result) | 171 | print(crawl_result) |
| 172 | ``` | ||
| 173 | |||
| 174 | ## v1 compatibility | ||
| 175 | |||
| 176 | For legacy code paths, v1 remains available under `firecrawl.v1` with the original method names. | ||
| 177 | |||
| 178 | ```python | ||
| 179 | from firecrawl import Firecrawl | ||
| 180 | |||
| 181 | firecrawl = Firecrawl(api_key="YOUR_API_KEY") | ||
| 182 | |||
| 183 | # v1 methods (feature‑frozen) | ||
| 184 | doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html']) | ||
| 185 | crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100) | ||
| 186 | map_v1 = firecrawl.v1.map_url('https://firecrawl.dev') | ||
| 171 | ``` | 187 | ``` |
| @@ -1,2 +1,5 @@ | |||
| 1 | API_URL=http://localhost:3002 | 1 | FIRECRAWL_API_KEY=<your api key> |
| 2 | API_KEY=fc-YOUR_API_KEY | 2 | FIRECRAWL_API_URL=https://api.firecrawl.dev |
| 3 | |||
| 4 | # TESTING PURPOSE: | ||
| 5 | IDMUX_URL= | ||
| @@ -17,7 +17,6 @@ if not os.getenv("API_URL"): · 16 unmodified lines | |||
| 17 | async def test_async_map_minimal(): | 17 | async def test_async_map_minimal(): |
| 18 | client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) | 18 | client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) |
| 19 | resp = await client.v2.map("https://docs.firecrawl.dev") | 19 | resp = await client.v2.map("https://docs.firecrawl.dev") |
| 20 | assert resp.success is True | ||
| 21 | assert hasattr(resp, "links") and isinstance(resp.links, list) | 20 | assert hasattr(resp, "links") and isinstance(resp.links, list) |
| 22 | if resp.links: | 21 | if resp.links: |
| 23 | first = resp.links[0] | 22 | first = resp.links[0] |
| @@ -36,7 +35,6 @@ async def test_async_map_with_all_params(sitemap): · 12 unmodified lines | |||
| 36 | sitemap=sitemap, | 35 | sitemap=sitemap, |
| 37 | timeout=15000, | 36 | timeout=15000, |
| 38 | ) | 37 | ) |
| 39 | assert resp.success is True | ||
| 40 | assert hasattr(resp, "links") and isinstance(resp.links, list) | 38 | assert hasattr(resp, "links") and isinstance(resp.links, list) |
| 41 | assert len(resp.links) <= 10 | 39 | assert len(resp.links) <= 10 |
| 42 | 40 | ||
| @@ -0,0 +1,73 @@ | |||
| 1 | import os | ||
| 2 | import json | ||
| 3 | import pytest | ||
| 4 | import requests | ||
| 5 | from dotenv import load_dotenv | ||
| 6 | |||
| 7 | load_dotenv() | ||
| 8 | |||
| 9 | def _idmux(identity_request: dict) -> dict: | ||
| 10 | idmux_url = os.getenv("IDMUX_URL") | ||
| 11 | if not idmux_url: | ||
| 12 | raise EnvironmentError("IDMUX_URL is not set. E2E tests must use idmux for credentials.") | ||
| 13 | run_number = int(os.getenv("GITHUB_RUN_NUMBER") or 0) | ||
| 14 | payload = { | ||
| 15 | "refName": os.getenv("GITHUB_REF_NAME") or "local", | ||
| 16 | "runNumber": run_number, | ||
| 17 | "concurrency": identity_request.get("concurrency", 100), | ||
| 18 | **identity_request, | ||
| 19 | } | ||
| 20 | resp = requests.post(idmux_url + "/", json=payload) | ||
| 21 | resp.raise_for_status() | ||
| 22 | return resp.json() | ||
| 23 | |||
| 24 | @pytest.fixture(scope="session") | ||
| 25 | def api_url(): | ||
| 26 | # Prefer TEST_URL, then FIRECRAWL_API_URL (for parity with JS), then legacy API_URL | ||
| 27 | return ( | ||
| 28 | os.getenv("TEST_URL") | ||
| 29 | or os.getenv("FIRECRAWL_API_URL") | ||
| 30 | or os.getenv("API_URL") | ||
| 31 | or "https://api.firecrawl.dev" | ||
| 32 | ) | ||
| 33 | |||
| 34 | # Resolve identity and export environment at import time so tests that read env at module import succeed | ||
| 35 | _IDENTITY = None | ||
| 36 | _API_URL = ( | ||
| 37 | os.getenv("TEST_URL") | ||
| 38 | or os.getenv("FIRECRAWL_API_URL") | ||
| 39 | or os.getenv("API_URL") | ||
| 40 | or "https://api.firecrawl.dev" | ||
| 41 | ) | ||
| 42 | |||
| 43 | _IDMUX_URL = os.getenv("IDMUX_URL") | ||
| 44 | if _IDMUX_URL: | ||
| 45 | run_name = os.getenv("PYTEST_RUN_NAME") or "py-e2e" | ||
| 46 | # If IDMUX_URL is set, idmux MUST succeed; do not silently fall back | ||
| 47 | _IDENTITY = _idmux({"name": run_name}) | ||
| 48 | os.environ["API_KEY"] = _IDENTITY.get("apiKey", "") | ||
| 49 | os.environ["API_URL"] = _API_URL | ||
| 50 | |||
| 51 | @pytest.fixture(scope="session") | ||
| 52 | def api_identity(): | ||
| 53 | return _IDENTITY or {"apiKey": os.getenv("API_KEY") or "", "teamId": os.getenv("TEST_TEAM_ID") or os.getenv("TEAM_ID") or ""} | ||
| 54 | |||
| 55 | @pytest.fixture(autouse=True) | ||
| 56 | def _inject_client(request, api_identity, api_url): | ||
| 57 | # For class-based tests that rely on self.client, inject a client if missing | ||
| 58 | inst = getattr(request, "instance", None) | ||
| 59 | if inst is not None and not hasattr(inst, "client"): | ||
| 60 | try: | ||
| 61 | from firecrawl import Firecrawl | ||
| 62 | inst.client = Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url) | ||
| 63 | except Exception: | ||
| 64 | pass | ||
| 65 | # For function-based modules that expect a module-level `firecrawl` symbol | ||
| 66 | mod = getattr(request, "module", None) | ||
| 67 | if mod is not None and not hasattr(mod, "firecrawl"): | ||
| 68 | try: | ||
| 69 | from firecrawl import Firecrawl | ||
| 70 | setattr(mod, "firecrawl", Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url)) | ||
| 71 | except Exception: | ||
| 72 | pass | ||
| 73 | |||
| @@ -24,7 +24,6 @@ class TestMapE2E: · 23 unmodified lines | |||
| 24 | def test_map_minimal_request(self): | 24 | def test_map_minimal_request(self): |
| 25 | resp = self.client.map("https://docs.firecrawl.dev") | 25 | resp = self.client.map("https://docs.firecrawl.dev") |
| 26 | 26 | ||
| 27 | assert hasattr(resp, "success") and resp.success is True | ||
| 28 | assert hasattr(resp, "links") and resp.links is not None | 27 | assert hasattr(resp, "links") and resp.links is not None |
| 29 | assert isinstance(resp.links, list) | 28 | assert isinstance(resp.links, list) |
| 30 | 29 | ||
| @@ -46,13 +45,11 @@ class TestMapE2E: · 15 unmodified lines | |||
| 46 | resp = self.client.map( | 45 | resp = self.client.map( |
| 47 | "https://docs.firecrawl.dev", | 46 | "https://docs.firecrawl.dev", |
| 48 | search="docs", | 47 | search="docs", |
| 49 | include_subdomains=True, | 48 | include_subdomains=True, limit=10, |
| 50 | limit=10, | ||
| 51 | sitemap=sitemap, | 49 | sitemap=sitemap, |
| 52 | timeout=15000, | 50 | timeout=15000, |
| 53 | ) | 51 | ) |
| 54 | 52 | ||
| 55 | assert hasattr(resp, "success") and resp.success is True | ||
| 56 | assert hasattr(resp, "links") and isinstance(resp.links, list) | 53 | assert hasattr(resp, "links") and isinstance(resp.links, list) |
| 57 | 54 | ||
| 58 | # Limit should be respected (server-side) | 55 | # Limit should be respected (server-side) |
| @@ -5,12 +5,6 @@ from firecrawl.types import SearchData, SearchResult, Document, ScrapeFormats, S · 4 unmodified lines | |||
| 5 | 5 | ||
| 6 | load_dotenv() | 6 | load_dotenv() |
| 7 | 7 | ||
| 8 | if not os.getenv("API_KEY"): | ||
| 9 | raise ValueError("API_KEY is not set") | ||
| 10 | |||
| 11 | if not os.getenv("API_URL"): | ||
| 12 | raise ValueError("API_URL is not set") | ||
| 13 | |||
| 14 | firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) | 8 | firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) |
| 15 | 9 | ||
| 16 | def _collect_texts(entries): | 10 | def _collect_texts(entries): |
| @@ -4,15 +4,10 @@ from firecrawl import Firecrawl · 3 unmodified lines | |||
| 4 | 4 | ||
| 5 | load_dotenv() | 5 | load_dotenv() |
| 6 | 6 | ||
| 7 | if not os.getenv("API_KEY"): | ||
| 8 | raise ValueError("API_KEY is not set") | ||
| 9 | |||
| 10 | if not os.getenv("API_URL"): | ||
| 11 | raise ValueError("API_URL is not set") | ||
| 12 | |||
| 13 | 7 | ||
| 14 | class TestUsageE2E: | 8 | class TestUsageE2E: |
| 15 | def setup_method(self): | 9 | def setup_method(self): |
| 10 | # Environment is exported by conftest at import time | ||
| 16 | self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) | 11 | self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) |
| 17 | 12 | ||
| 18 | def test_get_concurrency(self): | 13 | def test_get_concurrency(self): |
| @@ -14,6 +14,7 @@ if not os.getenv("API_URL"): · 13 unmodified lines | |||
| 14 | 14 | ||
| 15 | class TestWatcherE2E: | 15 | class TestWatcherE2E: |
| 16 | def setup_method(self): | 16 | def setup_method(self): |
| 17 | from firecrawl import Firecrawl | ||
| 17 | self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) | 18 | self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL")) |
| 18 | 19 | ||
| 19 | def test_crawl_watcher(self): | 20 | def test_crawl_watcher(self): |
| @@ -113,20 +113,18 @@ class AsyncV2Proxy: · 112 unmodified lines | |||
| 113 | 113 | ||
| 114 | class Firecrawl: | 114 | class Firecrawl: |
| 115 | """ | 115 | """ |
| 116 | Unified Firecrawl client with version-specific access patterns. | 116 | Unified Firecrawl client (v2 by default, v1 under ``.v1``). |
| 117 | 117 | ||
| 118 | This class provides a unified interface that allows users to access | 118 | Provides a single entrypoint that exposes the latest API directly while |
| 119 | different API versions through version-specific attributes (v1, v2) | 119 | keeping a feature-frozen v1 available for incremental migration. |
| 120 | while defaulting to v2 for direct method calls. | ||
| 121 | """ | 120 | """ |
| 122 | 121 | ||
| 123 | def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"): | 122 | def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"): |
| 124 | """ | 123 | """Initialize the unified client. |
| 125 | Initialize the unified Firecrawl client. | 124 | |
| 126 | |||
| 127 | Args: | 125 | Args: |
| 128 | api_key: Your Firecrawl API key | 126 | api_key: Firecrawl API key (or set ``FIRECRAWL_API_KEY``) |
| 129 | api_url: The Firecrawl API URL (defaults to production) | 127 | api_url: Base API URL (defaults to production) |
| 130 | """ | 128 | """ |
| 131 | self.api_key = api_key | 129 | self.api_key = api_key |
| 132 | self.api_url = api_url | 130 | self.api_url = api_url |
| @@ -164,13 +162,7 @@ class Firecrawl: · 31 unmodified lines | |||
| 164 | self.watcher = self._v2_client.watcher | 162 | self.watcher = self._v2_client.watcher |
| 165 | 163 | ||
| 166 | class AsyncFirecrawl: | 164 | class AsyncFirecrawl: |
| 167 | """ | 165 | """Async unified Firecrawl client (v2 by default, v1 under ``.v1``).""" |
| 168 | Unified Firecrawl client with version-specific access patterns. | ||
| 169 | |||
| 170 | This class provides a unified interface that allows users to access | ||
| 171 | different API versions through version-specific attributes (v1, v2) | ||
| 172 | while defaulting to v2 for direct method calls. | ||
| 173 | """ | ||
| 174 | 166 | ||
| 175 | def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"): | 167 | def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"): |
| 176 | self.api_key = api_key | 168 | self.api_key = api_key |
| @@ -185,6 +177,36 @@ class AsyncFirecrawl: · 8 unmodified lines | |||
| 185 | self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None | 177 | self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None |
| 186 | self.v2 = AsyncV2Proxy(self._v2_client) | 178 | self.v2 = AsyncV2Proxy(self._v2_client) |
| 187 | 179 | ||
| 180 | # Expose v2 async surface directly on the top-level client for ergonomic access | ||
| 181 | # Keep method names aligned with the sync client | ||
| 182 | self.scrape = self._v2_client.scrape | ||
| 183 | self.search = self._v2_client.search | ||
| 184 | self.map = self._v2_client.map | ||
| 185 | |||
| 186 | self.start_crawl = self._v2_client.start_crawl | ||
| 187 | self.get_crawl_status = self._v2_client.get_crawl_status | ||
| 188 | self.cancel_crawl = self._v2_client.cancel_crawl | ||
| 189 | self.crawl = self._v2_client.crawl | ||
| 190 | self.get_crawl_errors = self._v2_client.get_crawl_errors | ||
| 191 | self.active_crawls = self._v2_client.active_crawls | ||
| 192 | self.crawl_params_preview = self._v2_client.crawl_params_preview | ||
| 193 | |||
| 194 | self.start_batch_scrape = self._v2_client.start_batch_scrape | ||
| 195 | self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status | ||
| 196 | self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape | ||
| 197 | self.batch_scrape = self._v2_client.batch_scrape | ||
| 198 | self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors | ||
| 199 | |||
| 200 | self.start_extract = self._v2_client.start_extract | ||
| 201 | self.get_extract_status = self._v2_client.get_extract_status | ||
| 202 | self.extract = self._v2_client.extract | ||
| 203 | |||
| 204 | self.get_concurrency = self._v2_client.get_concurrency | ||
| 205 | self.get_credit_usage = self._v2_client.get_credit_usage | ||
| 206 | self.get_token_usage = self._v2_client.get_token_usage | ||
| 207 | |||
| 208 | self.watcher = self._v2_client.watcher | ||
| 209 | |||
| 188 | # Export Firecrawl as an alias for FirecrawlApp | 210 | # Export Firecrawl as an alias for FirecrawlApp |
| 189 | FirecrawlApp = Firecrawl | 211 | FirecrawlApp = Firecrawl |
| 190 | AsyncFirecrawlApp = AsyncFirecrawl | 212 | AsyncFirecrawlApp = AsyncFirecrawl |
File renamed without changes.
| @@ -408,8 +408,18 @@ class FirecrawlClient: · 407 unmodified lines | |||
| 408 | sitemap: Optional[Literal["only", "include", "skip"]] = None, | 408 | sitemap: Optional[Literal["only", "include", "skip"]] = None, |
| 409 | timeout: Optional[int] = None, | 409 | timeout: Optional[int] = None, |
| 410 | ) -> MapData: | 410 | ) -> MapData: |
| 411 | """ | 411 | """Map a URL and return discovered links. |
| 412 | Map a URL and return discovered links (with optional titles/descriptions). | 412 | |
| 413 | Args: | ||
| 414 | url: Root URL to explore | ||
| 415 | search: Optional substring filter for discovered links | ||
| 416 | include_subdomains: Whether to include subdomains | ||
| 417 | limit: Maximum number of links to return | ||
| 418 | sitemap: Sitemap usage mode ("only" | "include" | "skip") | ||
| 419 | timeout: Request timeout in milliseconds | ||
| 420 | |||
| 421 | Returns: | ||
| 422 | MapData containing the discovered links | ||
| 413 | """ | 423 | """ |
| 414 | options = MapOptions( | 424 | options = MapOptions( |
| 415 | search=search, | 425 | search=search, |
| @@ -434,8 +444,14 @@ class FirecrawlClient: · 18 unmodified lines | |||
| 434 | return crawl_module.cancel_crawl(self.http_client, crawl_id) | 444 | return crawl_module.cancel_crawl(self.http_client, crawl_id) |
| 435 | 445 | ||
| 436 | def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData: | 446 | def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData: |
| 437 | """ | 447 | """Derive crawl parameters from natural-language prompt. |
| 438 | Get crawl parameters from LLM based on URL and prompt. | 448 | |
| 449 | Args: | ||
| 450 | url: Root URL | ||
| 451 | prompt: Instruction describing how to crawl | ||
| 452 | |||
| 453 | Returns: | ||
| 454 | CrawlParamsData with normalized crawl configuration | ||
| 439 | """ | 455 | """ |
| 440 | request = CrawlParamsRequest(url=url, prompt=prompt) | 456 | request = CrawlParamsRequest(url=url, prompt=prompt) |
| 441 | return crawl_module.crawl_params_preview(self.http_client, request) | 457 | return crawl_module.crawl_params_preview(self.http_client, request) |
| @@ -453,8 +469,21 @@ class FirecrawlClient: · 11 unmodified lines | |||
| 453 | scrape_options: Optional['ScrapeOptions'] = None, | 469 | scrape_options: Optional['ScrapeOptions'] = None, |
| 454 | ignore_invalid_urls: Optional[bool] = None, | 470 | ignore_invalid_urls: Optional[bool] = None, |
| 455 | ): | 471 | ): |
| 456 | """ | 472 | """Start an extract job (non-blocking). |
| 457 | Start an extract job (non-blocking). Use get_extract_status to poll. | 473 | |
| 474 | Args: | ||
| 475 | urls: URLs to extract from (optional) | ||
| 476 | prompt: Natural-language instruction for extraction | ||
| 477 | schema: Target JSON schema for the output | ||
| 478 | system_prompt: Optional system instruction | ||
| 479 | allow_external_links: Allow hyperlinks in output | ||
| 480 | enable_web_search: Whether to augment with web search | ||
| 481 | show_sources: Include per-field/source mapping when available | ||
| 482 | scrape_options: Scrape options applied prior to extraction | ||
| 483 | ignore_invalid_urls: Skip invalid URLs instead of failing | ||
| 484 | |||
| 485 | Returns: | ||
| 486 | Response payload with job id/status (poll with get_extract_status) | ||
| 458 | """ | 487 | """ |
| 459 | return extract_module.start_extract( | 488 | return extract_module.start_extract( |
| 460 | self.http_client, | 489 | self.http_client, |
| @@ -484,8 +513,23 @@ class FirecrawlClient: · 23 unmodified lines | |||
| 484 | poll_interval: int = 2, | 513 | poll_interval: int = 2, |
| 485 | timeout: Optional[int] = None, | 514 | timeout: Optional[int] = None, |
| 486 | ): | 515 | ): |
| 487 | """ | 516 | """Extract structured data and wait until completion. |
| 488 | Extract structured data using the v2 client surface, waiting until completion. | 517 | |
| 518 | Args: | ||
| 519 | urls: URLs to extract from (optional) | ||
| 520 | prompt: Natural-language instruction for extraction | ||
| 521 | schema: Target JSON schema for the output | ||
| 522 | system_prompt: Optional system instruction | ||
| 523 | allow_external_links: Allow hyperlinks in output | ||
| 524 | enable_web_search: Whether to augment with web search | ||
| 525 | show_sources: Include per-field/source mapping when available | ||
| 526 | scrape_options: Scrape options applied prior to extraction | ||
| 527 | ignore_invalid_urls: Skip invalid URLs instead of failing | ||
| 528 | poll_interval: Seconds between status checks | ||
| 529 | timeout: Maximum seconds to wait (None for no timeout) | ||
| 530 | |||
| 531 | Returns: | ||
| 532 | Final extract response when completed | ||
| 489 | """ | 533 | """ |
| 490 | return extract_module.extract( | 534 | return extract_module.extract( |
| 491 | self.http_client, | 535 | self.http_client, |
| @@ -533,8 +577,39 @@ class FirecrawlClient: · 41 unmodified lines | |||
| 533 | integration: Optional[str] = None, | 577 | integration: Optional[str] = None, |
| 534 | idempotency_key: Optional[str] = None, | 578 | idempotency_key: Optional[str] = None, |
| 535 | ): | 579 | ): |
| 536 | """ | 580 | """Start a batch scrape job over multiple URLs (non-blocking). |
| 537 | Start a batch scrape job over multiple URLs. | 581 | |
| 582 | Args: | ||
| 583 | urls: List of URLs to scrape | ||
| 584 | formats: Output formats to collect per URL | ||
| 585 | headers: HTTP headers | ||
| 586 | include_tags: HTML tags to include | ||
| 587 | exclude_tags: HTML tags to exclude | ||
| 588 | only_main_content: Restrict scraping to main content | ||
| 589 | timeout: Per-request timeout in milliseconds | ||
| 590 | wait_for: Wait condition in milliseconds | ||
| 591 | mobile: Emulate mobile viewport | ||
| 592 | parsers: Parser list (e.g., ["pdf"]) | ||
| 593 | actions: Browser actions to perform | ||
| 594 | location: Location settings | ||
| 595 | skip_tls_verification: Skip TLS verification | ||
| 596 | remove_base64_images: Remove base64 images from output | ||
| 597 | fast_mode: Prefer faster scraping modes | ||
| 598 | use_mock: Use a mock data source (internal/testing) | ||
| 599 | block_ads: Block ads during scraping | ||
| 600 | proxy: Proxy setting | ||
| 601 | max_age: Cache max age | ||
| 602 | store_in_cache: Whether to store results in cache | ||
| 603 | webhook: Webhook configuration | ||
| 604 | append_to_id: Append to an existing batch job | ||
| 605 | ignore_invalid_urls: Skip invalid URLs without failing | ||
| 606 | max_concurrency: Max concurrent scrapes | ||
| 607 | zero_data_retention: Delete data after 24 hours | ||
| 608 | integration: Integration tag/name | ||
| 609 | idempotency_key: Header used to deduplicate starts | ||
| 610 | |||
| 611 | Returns: | ||
| 612 | Response payload with job id (poll with get_batch_scrape_status) | ||
| 538 | """ | 613 | """ |
| 539 | options = ScrapeOptions( | 614 | options = ScrapeOptions( |
| 540 | **{k: v for k, v in dict( | 615 | **{k: v for k, v in dict( |
| @@ -574,18 +649,47 @@ class FirecrawlClient: · 33 unmodified lines | |||
| 574 | ) | 649 | ) |
| 575 | 650 | ||
| 576 | def get_batch_scrape_status(self, job_id: str): | 651 | def get_batch_scrape_status(self, job_id: str): |
| 577 | """Get current status and any scraped data for a batch scrape job.""" | 652 | """Get current status and any scraped data for a batch job. |
| 653 | |||
| 654 | Args: | ||
| 655 | job_id: Batch job ID | ||
| 656 | |||
| 657 | Returns: | ||
| 658 | Status payload including counts and partial data | ||
| 659 | """ | ||
| 578 | return batch_module.get_batch_scrape_status(self.http_client, job_id) | 660 | return batch_module.get_batch_scrape_status(self.http_client, job_id) |
| 579 | 661 | ||
| 580 | def cancel_batch_scrape(self, job_id: str) -> bool: | 662 | def cancel_batch_scrape(self, job_id: str) -> bool: |
| 581 | """Cancel a running batch scrape job.""" | 663 | """Cancel a running batch scrape job. |
| 664 | |||
| 665 | Args: | ||
| 666 | job_id: Batch job ID | ||
| 667 | |||
| 668 | Returns: | ||
| 669 | True if the job was cancelled | ||
| 670 | """ | ||
| 582 | return batch_module.cancel_batch_scrape(self.http_client, job_id) | 671 | return batch_module.cancel_batch_scrape(self.http_client, job_id) |
| 583 | 672 | ||
| 584 | def get_batch_scrape_errors(self, job_id: str): | 673 | def get_batch_scrape_errors(self, job_id: str): |
| 585 | """Retrieve error details for a batch scrape job.""" | 674 | """Retrieve error details for a batch scrape job. |
| 675 | |||
| 676 | Args: | ||
| 677 | job_id: Batch job ID | ||
| 678 | |||
| 679 | Returns: | ||
| 680 | Errors and robots-blocked URLs for the job | ||
| 681 | """ | ||
| 586 | return batch_methods.get_batch_scrape_errors(self.http_client, job_id) | 682 | return batch_methods.get_batch_scrape_errors(self.http_client, job_id) |
| 587 | 683 | ||
| 588 | def get_extract_status(self, job_id: str): | 684 | def get_extract_status(self, job_id: str): |
| 685 | """Get the current status (and data if completed) of an extract job. | ||
| 686 | |||
| 687 | Args: | ||
| 688 | job_id: Extract job ID | ||
| 689 | |||
| 690 | Returns: | ||
| 691 | Extract response payload with status and optional data | ||
| 692 | """ | ||
| 589 | return extract_module.get_extract_status(self.http_client, job_id) | 693 | return extract_module.get_extract_status(self.http_client, job_id) |
| 590 | 694 | ||
| 591 | def get_concurrency(self): | 695 | def get_concurrency(self): |
| @@ -608,7 +712,17 @@ class FirecrawlClient: · 16 unmodified lines | |||
| 608 | poll_interval: int = 2, | 712 | poll_interval: int = 2, |
| 609 | timeout: Optional[int] = None, | 713 | timeout: Optional[int] = None, |
| 610 | ) -> Watcher: | 714 | ) -> Watcher: |
| 611 | """Create a polling-based watcher for crawl or batch jobs.""" | 715 | """Create a watcher for crawl or batch jobs. |
| 716 | |||
| 717 | Args: | ||
| 718 | job_id: Job ID to watch | ||
| 719 | kind: Job kind ("crawl" or "batch") | ||
| 720 | poll_interval: Seconds between status checks | ||
| 721 | timeout: Maximum seconds to watch (None for no timeout) | ||
| 722 | |||
| 723 | Returns: | ||
| 724 | Watcher instance | ||
| 725 | """ | ||
| 612 | return Watcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout) | 726 | return Watcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout) |
| 613 | 727 | ||
| 614 | def batch_scrape( | 728 | def batch_scrape( |
| @@ -160,8 +160,7 @@ async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse: · 159 unmodified lines | |||
| 160 | body = response.json() | 160 | body = response.json() |
| 161 | if not body.get("success"): | 161 | if not body.get("success"): |
| 162 | raise Exception(body.get("error", "Unknown error occurred")) | 162 | raise Exception(body.get("error", "Unknown error occurred")) |
| 163 | data = body.get("data", {}) | 163 | crawls_in = body.get("crawls", []) |
| 164 | crawls_in = data.get("crawls", []) | ||
| 165 | normalized = [] | 164 | normalized = [] |
| 166 | for c in crawls_in: | 165 | for c in crawls_in: |
| 167 | if isinstance(c, dict): | 166 | if isinstance(c, dict): |
| @@ -18,7 +18,7 @@ async def get_concurrency(client: AsyncHttpClient) -> ConcurrencyCheck: · 17 unmodified lines | |||
| 18 | 18 | ||
| 19 | 19 | ||
| 20 | async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage: | 20 | async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage: |
| 21 | resp = await client.get("/v2/team/credit-usage") | 21 | resp = await client.get("/v2/credit-usage") |
| 22 | if resp.status_code >= 400: | 22 | if resp.status_code >= 400: |
| 23 | handle_response_error(resp, "get credit usage") | 23 | handle_response_error(resp, "get credit usage") |
| 24 | body = resp.json() | 24 | body = resp.json() |
| @@ -29,7 +29,7 @@ async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage: · 4 unmodified lines | |||
| 29 | 29 | ||
| 30 | 30 | ||
| 31 | async def get_token_usage(client: AsyncHttpClient) -> TokenUsage: | 31 | async def get_token_usage(client: AsyncHttpClient) -> TokenUsage: |
| 32 | resp = await client.get("/v2/team/token-usage") | 32 | resp = await client.get("/v2/token-usage") |
| 33 | if resp.status_code >= 400: | 33 | if resp.status_code >= 400: |
| 34 | handle_response_error(resp, "get token usage") | 34 | handle_response_error(resp, "get token usage") |
| 35 | body = resp.json() | 35 | body = resp.json() |