History

(python/js/ts-sdks): readmes, e2e tests w idmux etc all good

rafaelmmiller committed 1 year ago

Commit 537f6c41 parent f4d904631 files · +788 −226Browse files

Unified · Split

31 files
  1. apps/api/src/controllers/v2/extract-status.ts +94 −0
  2. apps/api/src/routes/v2.ts +16 −0
  3. apps/js-sdk/firecrawl/.env.example +4 −1
  4. apps/js-sdk/firecrawl/README.md +85 −78
  5. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts +8 −4
  6. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts +8 −7
  7. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts +8 −4
  8. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts +8 −7
  9. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts +9 −8
  10. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts +24 −14
  11. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts +8 −4
  12. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/utils/idmux.ts +58 −0
  13. apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts +10 −4
  14. apps/js-sdk/firecrawl/src/index.ts +21 −7
  15. apps/js-sdk/firecrawl/src/v1/index.ts +12 −1
  16. apps/js-sdk/firecrawl/src/v2/client.ts +113 −0
  17. apps/js-sdk/firecrawl/src/v2/types.ts +3 −3
  18. apps/python-sdk/.env.example +4 −1
  19. apps/python-sdk/README.md +44 −28
  20. apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example +5 −2
  21. apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_map.py +0 −2
  22. apps/python-sdk/firecrawl/__tests__/e2e/v2/conftest.py +73 −0
  23. apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py +1 −4
  24. apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py +0 −6
  25. apps/python-sdk/firecrawl/__tests__/e2e/v2/test_usage.py +1 −6
  26. apps/python-sdk/firecrawl/__tests__/e2e/v2/test_watcher.py +1 −0
  27. apps/python-sdk/firecrawl/client.py +39 −17
  28. apps/python-sdk/firecrawl/firecrawl.backup.py +0 −0
  29. apps/python-sdk/firecrawl/v2/client.py +128 −14
  30. apps/python-sdk/firecrawl/v2/methods/aio/crawl.py +1 −2
  31. apps/python-sdk/firecrawl/v2/methods/aio/usage.py +2 −2
apps/api/src/controllers/v2/extract-status.ts+94−0
@@ -0,0 +1,94 @@
1import { Response } from "express";
2import { RequestWithAuth } from "./types";
3import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
4import { DBJob, PseudoJob } from "./crawl-status";
5import { getExtractQueue } from "../../services/queue-service";
6import { ExtractResult } from "../../lib/extract/extraction-service";
7import { supabaseGetJobById } from "../../lib/supabase-jobs";
8
9export async function getExtractJob(id: string): Promise<PseudoJob<ExtractResult> | null> {
10 const [bullJob, dbJob] = await Promise.all([
11 getExtractQueue().getJob(id),
12 (process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
13 ]);
14
15 if (!bullJob && !dbJob) return null;
16
17 const data = dbJob?.docs ?? bullJob?.returnvalue?.data;
18
19 const job: PseudoJob<any> = {
20 id,
21 getState: bullJob ? bullJob.getState.bind(bullJob) : (() => dbJob!.success ? "completed" : "failed"),
22 returnvalue: data,
23 data: {
24 scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbJob!.page_options,
25 teamId: bullJob ? bullJob.data.teamId : dbJob!.team_id,
26 },
27 timestamp: bullJob ? bullJob.timestamp : new Date(dbJob!.date_added).valueOf(),
28 failedReason: (bullJob ? bullJob.failedReason : dbJob!.message) || undefined,
29 }
30
31 return job;
32}
33
34export async function extractStatusController(
35 req: RequestWithAuth<{ jobId: string }, any, any>,
36 res: Response,
37) {
38 const extract = await getExtract(req.params.jobId);
39
40 let status = extract?.status;
41
42 if (extract && extract.team_id !== req.auth.team_id) {
43 return res.status(404).json({
44 success: false,
45 error: "Extract job not found",
46 });
47 }
48
49 let data: ExtractResult | [] = [];
50
51 if (!extract || extract.status === "completed") {
52 const jobData = await getExtractJob(req.params.jobId);
53 if ((!jobData && !extract) || (jobData && jobData.data.teamId !== req.auth.team_id)) {
54 return res.status(404).json({
55 success: false,
56 error: "Extract job not found",
57 });
58 }
59
60 if (jobData) {
61 const jobStatus = await jobData.getState();
62
63 if (jobStatus === "completed") {
64 status = "completed";
65 } else if (jobStatus === "failed") {
66 status = "failed";
67 } else {
68 status = "processing";
69 }
70 }
71
72 if (!jobData?.returnvalue) {
73 // if we got in the split-second where the redis is updated but the bull isn't
74 // just pretend it's still processing - MG
75 status = "processing";
76 } else {
77 data = jobData.returnvalue ?? [];
78 }
79 }
80
81 return res.status(200).json({
82 success: status === "failed" ? false : true,
83 data,
84 status,
85 error: extract?.error ?? undefined,
86 expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),
87 steps: extract?.showSteps ? extract.steps : undefined,
88 llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,
89 sources: extract?.showSources ? extract.sources : undefined,
90 costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
91 sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
92 tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,
93 });
94}
apps/api/src/routes/v2.ts+16−0
@@ -16,6 +16,8 @@ import { tokenUsageController } from "../controllers/v2/token-usage"; · 15 unmodified lines
16import { crawlCancelController } from "../controllers/v2/crawl-cancel";16import { crawlCancelController } from "../controllers/v2/crawl-cancel";
17import { concurrencyCheckController } from "../controllers/v2/concurrency-check";17import { concurrencyCheckController } from "../controllers/v2/concurrency-check";
18import { crawlStatusWSController } from "../controllers/v2/crawl-status-ws";18import { crawlStatusWSController } from "../controllers/v2/crawl-status-ws";
19import { extractController } from "../controllers/v2/extract";
20import { extractStatusController } from "../controllers/v2/extract-status";
19import {21import {
20 authMiddleware,22 authMiddleware,
21 checkCreditsMiddleware,23 checkCreditsMiddleware,
@@ -122,6 +124,20 @@ v2Router.get( · 100 unmodified lines
122 wrap(crawlErrorsController),124 wrap(crawlErrorsController),
123);125);
124 126
127v2Router.post(
128 "/extract",
129 authMiddleware(RateLimiterMode.Extract),
130 checkCreditsMiddleware(),
131 wrap(extractController),
132);
133
134v2Router.get(
135 "/extract/:jobId",
136 authMiddleware(RateLimiterMode.ExtractStatus),
137 wrap(extractStatusController),
138);
139
140
125v2Router.get(141v2Router.get(
126 "/credit-usage",142 "/credit-usage",
127 authMiddleware(RateLimiterMode.CrawlStatus),143 authMiddleware(RateLimiterMode.CrawlStatus),
apps/js-sdk/firecrawl/.env.example+4−1
@@ -1,2 +1,5 @@
1FIRECRAWL_API_KEY=<your api key>1FIRECRAWL_API_KEY=<your api key>
2FIRECRAWL_API_URL=https://api.firecrawl.dev2FIRECRAWL_API_URL=https://api.firecrawl.dev
3
4# TESTING PURPOSE:
5IDMUX_URL=
apps/js-sdk/firecrawl/README.md+85−78
@@ -18,171 +18,178 @@ npm install @mendable/firecrawl-js · 17 unmodified lines
18Here's an example of how to use the SDK with error handling:18Here's an example of how to use the SDK with error handling:
19 19
20```js20```js
21import FirecrawlApp, { CrawlParams, CrawlStatusResponse } from '@mendable/firecrawl-js';21import Firecrawl from '@mendable/firecrawl-js';
22 22
23const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});23const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });
24 24
25// Scrape a website25// Scrape a website
26const scrapeResponse = await app.scrapeUrl('https://firecrawl.dev', {26const scrapeResponse = await app.scrape('https://firecrawl.dev', {
27 formats: ['markdown', 'html'],27 formats: ['markdown', 'html'],
28});28});
29console.log(scrapeResponse);
29 30
30if (scrapeResponse) {31// Crawl a website (waiter)
31 console.log(scrapeResponse)32const crawlResponse = await app.crawl('https://firecrawl.dev', {
32}
33
34// Crawl a website
35const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
36 limit: 100,33 limit: 100,
37 scrapeOptions: {34 scrapeOptions: { formats: ['markdown', 'html'] },
38 formats: ['markdown', 'html'],35 pollInterval: 2,
39 }36});
40})37console.log(crawlResponse);
41
42console.log(crawlResponse)
43```38```
44 39
45### Scraping a URL40### Scraping a URL
46 41
47To scrape a single URL with error handling, use the `scrapeUrl` method. It takes the URL as a parameter and returns the scraped data as a dictionary.42To scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.
48 43
49```js44```js
50const url = "https://example.com";45const url = 'https://example.com';
51const scrapedData = await app.scrapeUrl(url);46const scrapedData = await app.scrape(url);
52```47```
53 48
54### Crawling a Website49### Crawling a Website
55 50
56To crawl a website with error handling, use the `crawlUrl` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.51To crawl a website with error handling, use the `crawl` method. It takes the starting URL and optional parameters, including limits and per‑page `scrapeOptions`.
57 52
58```js53```js
59const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {54const crawlResponse = await app.crawl('https://firecrawl.dev', {
60 limit: 100,55 limit: 100,
61 scrapeOptions: {56 scrapeOptions: { formats: ['markdown', 'html'] },
62 formats: ['markdown', 'html'],57});
63 }
64})
65```58```
66 59
67 60
68### Asynchronous Crawl61### Asynchronous Crawl
69 62
70To initiate an asynchronous crawl of a website, utilize the AsyncCrawlURL method. This method requires the starting URL and optional parameters as inputs. The params argument enables you to define various settings for the asynchronous crawl, such as the maximum number of pages to crawl, permitted domains, and the output format. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the crawl.63To start an asynchronous crawl, use `startCrawl`. It returns a job ID you can poll with `getCrawlStatus`.
71 64
72```js65```js
73const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});66const start = await app.startCrawl('https://mendable.ai', {
67 excludePaths: ['blog/*'],
68 limit: 5,
69});
74```70```
75 71
76### Checking Crawl Status72### Checking Crawl Status
77 73
78To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job`74To check the status of a crawl job with error handling, use the `getCrawlStatus` method. It takes the job ID as a parameter and returns the current status.
79 75
80```js76```js
81const status = await app.checkCrawlStatus(id);77const status = await app.getCrawlStatus(id);
82```78```
83 79
84### Extracting structured data from a URL80### Extracting structured data from URLs
85 81
86With LLM extraction, you can easily extract structured data from any URL. We support zod schema to make it easier for you too. Here is how you to use it:82Use `extract` with a prompt and schema. Zod schemas are supported directly.
87 83
88```js84```js
89import FirecrawlApp from "@mendable/firecrawl-js";85import Firecrawl from '@mendable/firecrawl-js';
90import { z } from "zod";86import { z } from 'zod';
91 87
92const app = new FirecrawlApp({88const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });
93 apiKey: "fc-YOUR_API_KEY",
94});
95 89
96// Define schema to extract contents into
97const schema = z.object({90const schema = z.object({
98 top: z91 title: z.string(),
99 .array(
100 z.object({
101 title: z.string(),
102 points: z.number(),
103 by: z.string(),
104 commentsURL: z.string(),
105 })
106 )
107 .length(5)
108 .describe("Top 5 stories on Hacker News"),
109});92});
110 93
111const scrapeResult = await app.scrapeUrl("https://firecrawl.dev", {94const result = await app.extract({
112 extractorOptions: { extractionSchema: schema },95 urls: ['https://firecrawl.dev'],
96 prompt: 'Extract the page title',
97 schema,
98 showSources: true,
113});99});
114 100
115console.log(scrapeResult.data["llm_extraction"]);101console.log(result.data);
116```102```
117 103
118### Map a Website104### Map a Website
119 105
120Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap.106Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to utilize the sitemap or include subdomains.
121 107
122```js108```js
123const mapResult = await app.mapUrl('https://example.com') as MapResponse;109const mapResult = await app.map('https://example.com');
124console.log(mapResult)110console.log(mapResult);
125```111```
126 112
127### Crawl a website with WebSockets113### Crawl a website with real‑time updates
128 114
129To crawl a website with WebSockets, use the `crawlUrlAndWatch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.115To receive real‑time updates, start a crawl and attach a watcher.
130 116
131```js117```js
132// Crawl a website with WebSockets:118const start = await app.startCrawl('https://mendable.ai', { excludePaths: ['blog/*'], limit: 5 });
133const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5});119const watch = app.watcher(start.id, { kind: 'crawl', pollInterval: 2 });
134 120
135watch.addEventListener("document", doc => {121watch.on('document', (doc) => {
136 console.log("DOC", doc.detail);122 console.log('DOC', doc);
137});123});
138 124
139watch.addEventListener("error", err => {125watch.on('error', (err) => {
140 console.error("ERR", err.detail.error);126 console.error('ERR', err);
141});127});
142 128
143watch.addEventListener("done", state => {129watch.on('done', (state) => {
144 console.log("DONE", state.detail.status);130 console.log('DONE', state.status);
145});131});
132
133await watch.start();
146```134```
147 135
148### Batch scraping multiple URLs136### Batch scraping multiple URLs
149 137
150To batch scrape multiple URLs with error handling, use the `batchScrapeUrls` method. It takes the starting URLs and optional parameters as arguments. The `params` argument allows you to specify additional options for the batch scrape job, such as the output formats.138To batch scrape multiple URLs with error handling, use the `batchScrape` method.
151 139
152```js140```js
153const batchScrapeResponse = await app.batchScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], {141const batchScrapeResponse = await app.batchScrape(['https://firecrawl.dev', 'https://mendable.ai'], {
154 formats: ['markdown', 'html'],142 formats: ['markdown', 'html'],
155})143});
156```144```
157 145
158 146
159#### Asynchronous batch scrape147#### Asynchronous batch scrape
160 148
161To initiate an asynchronous batch scrape, utilize the `asyncBatchScrapeUrls` method. This method requires the starting URLs and optional parameters as inputs. The params argument enables you to define various settings for the scrape, such as the output formats. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the batch scrape.149To start an asynchronous batch scrape, use `startBatchScrape` and poll with `getBatchScrapeStatus`.
162 150
163```js151```js
164const asyncBatchScrapeResult = await app.asyncBatchScrapeUrls(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] });152const asyncBatchScrapeResult = await app.startBatchScrape(['https://firecrawl.dev', 'https://mendable.ai'], {
153 formats: ['markdown', 'html'],
154});
165```155```
166 156
167#### Batch scrape with WebSockets157#### Batch scrape with real‑time updates
168 158
169To use batch scrape with WebSockets, use the `batchScrapeUrlsAndWatch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the batch scrape job, such as the output formats.159To use batch scrape with real‑time updates, start the job and watch it using the watcher.
170 160
171```js161```js
172// Batch scrape multiple URLs with WebSockets:162const start = await app.startBatchScrape(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] });
173const watch = await app.batchScrapeUrlsAndWatch(['https://firecrawl.dev', 'https://mendable.ai'], { formats: ['markdown', 'html'] });163const watch = app.watcher(start.id, { kind: 'batch', pollInterval: 2 });
174 164
175watch.addEventListener("document", doc => {165watch.on('document', (doc) => {
176 console.log("DOC", doc.detail);166 console.log('DOC', doc);
177});167});
178 168
179watch.addEventListener("error", err => {169watch.on('error', (err) => {
180 console.error("ERR", err.detail.error);170 console.error('ERR', err);
181});171});
182 172
183watch.addEventListener("done", state => {173watch.on('done', (state) => {
184 console.log("DONE", state.detail.status);174 console.log('DONE', state.status);
185});175});
176
177await watch.start();
178```
179
180## v1 compatibility
181
182The feature‑frozen v1 is still available under `app.v1` with the original method names.
183
184```js
185import Firecrawl from '@mendable/firecrawl-js';
186
187const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY' });
188
189// v1 methods (feature‑frozen)
190const scrapeV1 = await app.v1.scrapeUrl('https://firecrawl.dev', { formats: ['markdown', 'html'] });
191const crawlV1 = await app.v1.crawlUrl('https://firecrawl.dev', { limit: 100 });
192const mapV1 = await app.v1.mapUrl('https://firecrawl.dev');
186```193```
187 194
188## Error Handling195## Error Handling
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/batch.test.ts+8−4
@@ -3,14 +3,18 @@ · 2 unmodified lines
3 */3 */
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import { config } from "dotenv";5import { config } from "dotenv";
6import { describe, test, expect } from "@jest/globals";6import { getIdentity, getApiUrl } from "./utils/idmux";
7import { describe, test, expect, beforeAll } from "@jest/globals";
7 8
8config();9config();
9 10
10const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";11const API_URL = getApiUrl();
11const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";12let client: Firecrawl;
12 13
13const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });14beforeAll(async () => {
15 const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
16 client = new Firecrawl({ apiKey, apiUrl: API_URL });
17});
14 18
15describe("v2.batch e2e", () => {19describe("v2.batch e2e", () => {
16 test("batch scrape minimal (wait)", async () => {20 test("batch scrape minimal (wait)", async () => {
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/crawl.test.ts+8−7
@@ -3,19 +3,20 @@ · 2 unmodified lines
3 */3 */
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import { config } from "dotenv";5import { config } from "dotenv";
6import { describe, test, expect } from "@jest/globals";6import { getIdentity, getApiUrl } from "./utils/idmux";
7import { describe, test, expect, beforeAll } from "@jest/globals";
7 8
8config();9config();
9 10
10const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";11const API_URL = getApiUrl();
11const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";12let client: Firecrawl;
12 13
13const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });14beforeAll(async () => {
15 const { apiKey } = await getIdentity({ name: "js-e2e-crawl" });
16 client = new Firecrawl({ apiKey, apiUrl: API_URL });
17});
14 18
15describe("v2.crawl e2e", () => {19describe("v2.crawl e2e", () => {
16 if (!client) {
17 console.warn("Skipping v2.crawl e2e: FIRECRAWL_API_KEY not set");
18 }
19 20
20 test("start crawl minimal request", async () => {21 test("start crawl minimal request", async () => {
21 if (!client) throw new Error();22 if (!client) throw new Error();
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/extract.test.ts+8−4
@@ -3,15 +3,19 @@ · 2 unmodified lines
3 */3 */
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import { config } from "dotenv";5import { config } from "dotenv";
6import { describe, test, expect } from "@jest/globals";6import { getIdentity, getApiUrl } from "./utils/idmux";
7import { describe, test, expect, beforeAll } from "@jest/globals";
7import { z } from "zod";8import { z } from "zod";
8 9
9config();10config();
10 11
11const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";12const API_URL = getApiUrl();
12const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";13let client: Firecrawl;
13 14
14const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });15beforeAll(async () => {
16 const { apiKey } = await getIdentity({ name: "js-e2e-extract" });
17 client = new Firecrawl({ apiKey, apiUrl: API_URL });
18});
15 19
16describe("v2.extract e2e", () => {20describe("v2.extract e2e", () => {
17 test("extract minimal with prompt", async () => {21 test("extract minimal with prompt", async () => {
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/map.test.ts+8−7
@@ -3,19 +3,20 @@ · 2 unmodified lines
3 */3 */
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import { config } from "dotenv";5import { config } from "dotenv";
6import { describe, test, expect } from "@jest/globals";6import { getIdentity, getApiUrl } from "./utils/idmux";
7import { describe, test, expect, beforeAll } from "@jest/globals";
7 8
8config();9config();
9 10
10const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";11const API_URL = getApiUrl();
11const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";12let client: Firecrawl;
12 13
13const client = API_KEY ? new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL }) : null as any;14beforeAll(async () => {
15 const { apiKey } = await getIdentity({ name: "js-e2e-map" });
16 client = new Firecrawl({ apiKey, apiUrl: API_URL });
17});
14 18
15describe("v2.map e2e", () => {19describe("v2.map e2e", () => {
16 if (!client) {
17 console.warn("Skipping v2.map e2e: FIRECRAWL_API_KEY not set");
18 }
19 20
20 test("minimal request", async () => {21 test("minimal request", async () => {
21 if (!client) throw new Error();22 if (!client) throw new Error();
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/scrape.test.ts+9−8
@@ -4,19 +4,20 @@ · 3 unmodified lines
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import { z } from "zod";5import { z } from "zod";
6import { config } from "dotenv";6import { config } from "dotenv";
7import { describe, test, expect } from "@jest/globals";7import { getIdentity, getApiUrl } from "./utils/idmux";
8import { describe, test, expect, beforeAll } from "@jest/globals";
8 9
9config();10config();
10 11
11const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";12const API_URL = getApiUrl();
12const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";13let client: Firecrawl;
13 14
14describe("v2.scrape e2e", () => {15beforeAll(async () => {
15 if (!API_KEY) {16 const { apiKey } = await getIdentity({ name: "js-e2e-scrape" });
16 console.warn("Skipping v2.scrape e2e: FIRECRAWL_API_KEY not set");17 client = new Firecrawl({ apiKey, apiUrl: API_URL });
17 }18});
18 19
19 const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL })20describe("v2.scrape e2e", () => {
20 21
21 const assertValidDocument = (doc: any) => {22 const assertValidDocument = (doc: any) => {
22 expect(doc).toBeTruthy();23 expect(doc).toBeTruthy();
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/search.test.ts+24−14
@@ -2,15 +2,20 @@ · 1 unmodified line
2 * E2E tests for v2 search (translated from Python tests)2 * E2E tests for v2 search (translated from Python tests)
3 */3 */
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import type { Document, SearchResult } from "../../../index";
5import { config } from "dotenv";6import { config } from "dotenv";
6import { describe, test, expect } from "@jest/globals";7import { getIdentity, getApiUrl } from "./utils/idmux";
8import { describe, test, expect, beforeAll } from "@jest/globals";
7 9
8config();10config();
9 11
10const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";12const API_URL = getApiUrl();
11const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";13let client: Firecrawl;
12 14
13const client = API_KEY ? new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL }) : null as any;15beforeAll(async () => {
16 const { apiKey } = await getIdentity({ name: "js-e2e-search" });
17 client = new Firecrawl({ apiKey, apiUrl: API_URL });
18});
14 19
15function collectTexts(entries: any[] | undefined): string[] {20function collectTexts(entries: any[] | undefined): string[] {
16 const texts: string[] = [];21 const texts: string[] = [];
@@ -23,14 +28,13 @@ function collectTexts(entries: any[] | undefined): string[] { · 6 unmodified lines
23 return texts;28 return texts;
24}29}
25 30
26function isDocument(entry: any): boolean {31function isDocument(entry: Document | SearchResult | undefined | null): entry is Document {
27 return !!entry && (typeof entry.markdown === 'string' || typeof entry.html === 'string');32 if (!entry) return false;
33 const d = entry as Document;
34 return typeof d.markdown === 'string' || typeof d.html === 'string';
28}35}
29 36
30describe("v2.search e2e", () => {37describe("v2.search e2e", () => {
31 if (!client) {
32 console.warn("Skipping v2.search e2e: FIRECRAWL_API_KEY not set");
33 }
34 38
35 test("minimal request", async () => {39 test("minimal request", async () => {
36 if (!client) throw new Error();40 if (!client) throw new Error();
@@ -72,8 +76,12 @@ describe("v2.search e2e", () => { · 35 unmodified lines
72 }76 }
73 expect(results.images == null).toBe(true);77 expect(results.images == null).toBe(true);
74 78
75 const webTitles = (results.web || []).map(r => (r?.title || "").toString().toLowerCase());79 const webTitles = (results.web || [])
76 const webDescriptions = (results.web || []).map(r => (r?.description || "").toString().toLowerCase());80 .filter((r): r is SearchResult => !isDocument(r))
81 .map(r => (r.title || "").toString().toLowerCase());
82 const webDescriptions = (results.web || [])
83 .filter((r): r is SearchResult => !isDocument(r))
84 .map(r => (r.description || "").toString().toLowerCase());
77 const allWebText = (webTitles.concat(webDescriptions)).join(" ");85 const allWebText = (webTitles.concat(webDescriptions)).join(" ");
78 expect(allWebText.includes("firecrawl")).toBe(true);86 expect(allWebText.includes("firecrawl")).toBe(true);
79 }, 90_000);87 }, 90_000);
@@ -174,8 +182,10 @@ describe("v2.search e2e", () => { · 94 unmodified lines
174 expect(results.images).toBeTruthy();182 expect(results.images).toBeTruthy();
175 expect((results.images || []).length).toBeLessThanOrEqual(3);183 expect((results.images || []).length).toBeLessThanOrEqual(3);
176 for (const result of results.images || []) {184 for (const result of results.images || []) {
177 expect(typeof result.url).toBe("string");185 if (!isDocument(result)) {
178 expect(result.url.startsWith("http")).toBe(true);186 expect(typeof result.url).toBe("string");
187 expect(result.url.startsWith("http")).toBe(true);
188 }
179 }189 }
180 }, 120_000);190 }, 120_000);
181 191
@@ -187,7 +197,7 @@ describe("v2.search e2e", () => { · 5 unmodified lines
187 });197 });
188 const results2 = await client.search("python programming", {198 const results2 = await client.search("python programming", {
189 limit: 1,199 limit: 1,
190 scrapeOptions: { formats: [{ type: "markdown" as any }] },200 scrapeOptions: { formats: ["markdown"] },
191 });201 });
192 expect(results1).toBeTruthy();202 expect(results1).toBeTruthy();
193 expect(results2).toBeTruthy();203 expect(results2).toBeTruthy();
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts+8−4
@@ -3,14 +3,18 @@ · 2 unmodified lines
3 */3 */
4import Firecrawl from "../../../index";4import Firecrawl from "../../../index";
5import { config } from "dotenv";5import { config } from "dotenv";
6import { describe, test, expect } from "@jest/globals";6import { getIdentity, getApiUrl } from "./utils/idmux";
7import { describe, test, expect, beforeAll } from "@jest/globals";
7 8
8config();9config();
9 10
10const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";11const API_URL = getApiUrl();
11const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";12let client: Firecrawl;
12 13
13const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });14beforeAll(async () => {
15 const { apiKey } = await getIdentity({ name: "js-e2e-usage" });
16 client = new Firecrawl({ apiKey, apiUrl: API_URL });
17});
14 18
15describe("v2.usage e2e", () => {19describe("v2.usage e2e", () => {
16 test("get_concurrency", async () => {20 test("get_concurrency", async () => {
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/utils/idmux.ts+58−0
@@ -0,0 +1,58 @@
1export type IdmuxRequest = {
2 name?: string;
3 concurrency?: number;
4 credits?: number;
5 tokens?: number;
6 teamId?: string;
7 flags?: Record<string, unknown>;
8};
9
10export type Identity = {
11 apiKey: string;
12 teamId: string;
13};
14
15let cachedIdentity: Identity | null = null;
16
17export function getApiUrl(): string {
18 return process.env.TEST_URL ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
19}
20
21export async function getIdentity(req: IdmuxRequest = {}): Promise<Identity> {
22 if (cachedIdentity) return cachedIdentity;
23
24 const idmuxUrl = process.env.IDMUX_URL;
25 if (!idmuxUrl) {
26 const fallback: Identity = {
27 apiKey: process.env.TEST_API_KEY ?? process.env.FIRECRAWL_API_KEY ?? "",
28 teamId: process.env.TEST_TEAM_ID ?? "",
29 };
30 cachedIdentity = fallback;
31 return fallback;
32 }
33
34 const runNumberRaw = process.env.GITHUB_RUN_NUMBER;
35 const runNumber = runNumberRaw ? Number(runNumberRaw) : 0;
36 const body = {
37 refName: process.env.GITHUB_REF_NAME ?? "local",
38 runNumber: Number.isFinite(runNumber) ? runNumber : 0,
39 concurrency: req.concurrency ?? 100,
40 ...req,
41 };
42
43 const res = await fetch(`${idmuxUrl}/`, {
44 method: "POST",
45 headers: { "Content-Type": "application/json" },
46 body: JSON.stringify(body),
47 });
48
49 if (!res.ok) {
50 const text = await res.text();
51 throw new Error(`idmux request failed: ${res.status} ${text}`);
52 }
53
54 const identity = (await res.json()) as Identity;
55 cachedIdentity = identity;
56 return identity;
57}
58
apps/js-sdk/firecrawl/src/__tests__/e2e/v2/watcher.test.ts+10−4
@@ -1,15 +1,21 @@
1import Firecrawl from "../../../index";1import Firecrawl from "../../../index";
2import { config } from "dotenv";2import { config } from "dotenv";
3import { describe, test, expect } from "@jest/globals";3import { describe, test, expect, beforeAll } from "@jest/globals";
4import { getIdentity } from "./utils/idmux";
4 5
5config();6config();
6 7
7const API_KEY = process.env.FIRECRAWL_API_KEY ?? "";
8const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";8const API_URL = process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev";
9let client: Firecrawl;
10
11beforeAll(async () => {
12 const { apiKey } = await getIdentity({ name: "js-e2e-watcher" });
13 client = new Firecrawl({ apiKey, apiUrl: API_URL });
14});
9 15
10describe("v2.watcher e2e", () => {16describe("v2.watcher e2e", () => {
11 test("crawl watcher minimal", async () => {17 test("crawl watcher minimal", async () => {
12 const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });18 // client is initialized in beforeAll
13 const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });19 const start = await client.startCrawl("https://docs.firecrawl.dev", { limit: 3 });
14 20
15 expect(typeof start.id).toBe("string");21 expect(typeof start.id).toBe("string");
@@ -49,7 +55,7 @@ describe("v2.watcher e2e", () => { · 33 unmodified lines
49 }, 240_000);55 }, 240_000);
50 56
51 test("batch watcher with options (kind, pollInterval, timeout)", async () => {57 test("batch watcher with options (kind, pollInterval, timeout)", async () => {
52 const client = new Firecrawl({ apiKey: API_KEY, apiUrl: API_URL });58 // client is initialized in beforeAll
53 const urls = [59 const urls = [
54 "https://docs.firecrawl.dev",60 "https://docs.firecrawl.dev",
55 "https://firecrawl.dev",61 "https://firecrawl.dev",
apps/js-sdk/firecrawl/src/index.ts+21−7
@@ -1,23 +1,37 @@
1// Unified entrypoint for Firecrawl JS SDK1/**
2// - Default export: unified Firecrawl (v2 by default, v1 under .v1)2 * Firecrawl JS/TS SDK — unified entrypoint.
3// - Named export FirecrawlClient: direct v2 client3 * - v2 by default on the top‑level client
4// - Named export FirecrawlAppV1: legacy v1 class4 * - v1 available under `.v1` (feature‑frozen)
5// - Re-export v2 types5 * - Exports: `Firecrawl` (default), `FirecrawlClient` (v2), `FirecrawlAppV1` (v1), and v2 types
6 */
6 7
8/** Direct v2 client. */
7export { FirecrawlClient } from "./v2/client";9export { FirecrawlClient } from "./v2/client";
10/** Public v2 request/response types. */
8export * from "./v2/types";11export * from "./v2/types";
12/** Legacy v1 client (feature‑frozen). */
9export { default as FirecrawlAppV1 } from "./v1";13export { default as FirecrawlAppV1 } from "./v1";
10 14
11import V1 from "./v1";15import V1 from "./v1";
12import { FirecrawlClient as V2 } from "./v2/client";16import { FirecrawlClient as V2 } from "./v2/client";
13import type { FirecrawlAppConfig } from "./v1";17import type { FirecrawlAppConfig } from "./v1";
14 18
19/** Unified client: extends v2 and adds `.v1` for backward compatibility. */
15export class Firecrawl extends V2 {20export class Firecrawl extends V2 {
16 public v1: V1;21 /** Feature‑frozen v1 client (lazy). */
22 private _v1?: V1;
23 private _v1Opts: FirecrawlAppConfig;
17 24
25 /** @param opts API credentials and base URL. */
18 constructor(opts: FirecrawlAppConfig = {}) {26 constructor(opts: FirecrawlAppConfig = {}) {
19 super(opts as any);27 super(opts as any);
20 this.v1 = new V1(opts);28 this._v1Opts = opts;
29 }
30
31 /** Access the legacy v1 client (instantiated on first access). */
32 get v1(): V1 {
33 if (!this._v1) this._v1 = new V1(this._v1Opts);
34 return this._v1;
21 }35 }
22}36}
23 37
apps/js-sdk/firecrawl/src/v1/index.ts+12−1
@@ -582,10 +582,21 @@ export default class FirecrawlApp { · 581 unmodified lines
582 582
583 private async getVersion(): Promise<string> {583 private async getVersion(): Promise<string> {
584 try {584 try {
585 // Prefer env-provided version when available (works well in test runners)
586 if (typeof process !== 'undefined' && process.env && process.env.npm_package_version) {
587 return process.env.npm_package_version as string;
588 }
585 const packageJson = await import('../../package.json', { assert: { type: 'json' } });589 const packageJson = await import('../../package.json', { assert: { type: 'json' } });
586 return packageJson.default.version;590 return packageJson.default.version;
587 } catch (error) {591 } catch (error) {
588 console.error("Error getting version:", error);592 // Suppress noisy logs under test environments
593 const isTest = typeof process !== 'undefined' && (
594 process.env.JEST_WORKER_ID != null || process.env.NODE_ENV === 'test'
595 );
596 if (!isTest) {
597 // eslint-disable-next-line no-console
598 console.error("Error getting version:", error);
599 }
589 return "1.25.1";600 return "1.25.1";
590 }601 }
591 }602 }
apps/js-sdk/firecrawl/src/v2/client.ts+113−0
@@ -38,17 +38,32 @@ import type { · 37 unmodified lines
38import { Watcher } from "./watcher";38import { Watcher } from "./watcher";
39import type { WatcherOptions } from "./watcher";39import type { WatcherOptions } from "./watcher";
40 40
41/**
42 * Configuration for the v2 client transport.
43 */
41export interface FirecrawlClientOptions {44export interface FirecrawlClientOptions {
45 /** API key (falls back to FIRECRAWL_API_KEY). */
42 apiKey?: string | null;46 apiKey?: string | null;
47 /** API base URL (falls back to FIRECRAWL_API_URL or https://api.firecrawl.dev). */
43 apiUrl?: string | null;48 apiUrl?: string | null;
49 /** Per-request timeout in milliseconds (optional). */
44 timeoutMs?: number;50 timeoutMs?: number;
51 /** Max automatic retries for transient failures (optional). */
45 maxRetries?: number;52 maxRetries?: number;
53 /** Exponential backoff factor for retries (optional). */
46 backoffFactor?: number;54 backoffFactor?: number;
47}55}
48 56
57/**
58 * Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
59 */
49export class FirecrawlClient {60export class FirecrawlClient {
50 private readonly http: HttpClient;61 private readonly http: HttpClient;
51 62
63 /**
64 * Create a v2 client.
65 * @param options Transport configuration (API key, base URL, timeouts, retries).
66 */
52 constructor(options: FirecrawlClientOptions = {}) {67 constructor(options: FirecrawlClientOptions = {}) {
53 const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";68 const apiKey = options.apiKey ?? process.env.FIRECRAWL_API_KEY ?? "";
54 const apiUrl = (options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");69 const apiUrl = (options.apiUrl ?? process.env.FIRECRAWL_API_URL ?? "https://api.firecrawl.dev").replace(/\/$/, "");
@@ -65,83 +80,181 @@ export class FirecrawlClient { · 10 unmodified lines
65 }80 }
66 81
67 // Scrape82 // Scrape
83 /**
84 * Scrape a single URL.
85 * @param url Target URL.
86 * @param options Optional scrape options (formats, headers, etc.).
87 * @returns Resolved document with requested formats.
88 */
68 async scrape(url: string, options?: ScrapeOptions): Promise<Document> {89 async scrape(url: string, options?: ScrapeOptions): Promise<Document> {
69 return scrape(this.http, url, options);90 return scrape(this.http, url, options);
70 }91 }
71 92
72 // Search93 // Search
94 /**
95 * Search the web and optionally scrape each result.
96 * @param query Search query string.
97 * @param req Additional search options (sources, limit, scrapeOptions, etc.).
98 * @returns Structured search results.
99 */
73 async search(query: string, req: Omit<SearchRequest, "query"> = {}): Promise<SearchData> {100 async search(query: string, req: Omit<SearchRequest, "query"> = {}): Promise<SearchData> {
74 return search(this.http, { query, ...req });101 return search(this.http, { query, ...req });
75 }102 }
76 103
77 // Map104 // Map
105 /**
106 * Map a site to discover URLs (sitemap-aware).
107 * @param url Root URL to map.
108 * @param options Mapping options (sitemap mode, includeSubdomains, limit, timeout).
109 * @returns Discovered links.
110 */
78 async map(url: string, options?: MapOptions): Promise<MapData> {111 async map(url: string, options?: MapOptions): Promise<MapData> {
79 return mapMethod(this.http, url, options);112 return mapMethod(this.http, url, options);
80 }113 }
81 114
82 // Crawl115 // Crawl
116 /**
117 * Start a crawl job (async).
118 * @param url Root URL to crawl.
119 * @param req Crawl configuration (paths, limits, scrapeOptions, webhook, etc.).
120 * @returns Job id and url.
121 */
83 async startCrawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> = {}): Promise<CrawlResponse> {122 async startCrawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> = {}): Promise<CrawlResponse> {
84 return startCrawl(this.http, { url, ...(req as any) });123 return startCrawl(this.http, { url, ...(req as any) });
85 }124 }
125 /**
126 * Get the status and partial data of a crawl job.
127 * @param jobId Crawl job id.
128 */
86 async getCrawlStatus(jobId: string): Promise<CrawlJob> {129 async getCrawlStatus(jobId: string): Promise<CrawlJob> {
87 return getCrawlStatus(this.http, jobId);130 return getCrawlStatus(this.http, jobId);
88 }131 }
132 /**
133 * Cancel a crawl job.
134 * @param jobId Crawl job id.
135 * @returns True if cancelled.
136 */
89 async cancelCrawl(jobId: string): Promise<boolean> {137 async cancelCrawl(jobId: string): Promise<boolean> {
90 return cancelCrawl(this.http, jobId);138 return cancelCrawl(this.http, jobId);
91 }139 }
140 /**
141 * Convenience waiter: start a crawl and poll until it finishes.
142 * @param url Root URL to crawl.
143 * @param req Crawl configuration plus waiter controls (pollInterval, timeout seconds).
144 * @returns Final job snapshot.
145 */
92 async crawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {146 async crawl(url: string, req: Omit<Parameters<typeof startCrawl>[1], "url"> & { pollInterval?: number; timeout?: number } = {}): Promise<CrawlJob> {
93 return crawlWaiter(this.http, { url, ...(req as any) }, req.pollInterval, req.timeout);147 return crawlWaiter(this.http, { url, ...(req as any) }, req.pollInterval, req.timeout);
94 }148 }
149 /**
150 * Retrieve crawl errors and robots.txt blocks.
151 * @param crawlId Crawl job id.
152 */
95 async getCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse> {153 async getCrawlErrors(crawlId: string): Promise<CrawlErrorsResponse> {
96 return getCrawlErrors(this.http, crawlId);154 return getCrawlErrors(this.http, crawlId);
97 }155 }
156 /**
157 * List active crawls for the authenticated team.
158 */
98 async getActiveCrawls(): Promise<ActiveCrawlsResponse> {159 async getActiveCrawls(): Promise<ActiveCrawlsResponse> {
99 return getActiveCrawls(this.http);160 return getActiveCrawls(this.http);
100 }161 }
162 /**
163 * Preview normalized crawl parameters produced by a natural-language prompt.
164 * @param url Root URL.
165 * @param prompt Natural-language instruction.
166 */
101 async crawlParamsPreview(url: string, prompt: string): Promise<Record<string, unknown>> {167 async crawlParamsPreview(url: string, prompt: string): Promise<Record<string, unknown>> {
102 return crawlParamsPreview(this.http, url, prompt);168 return crawlParamsPreview(this.http, url, prompt);
103 }169 }
104 170
105 // Batch171 // Batch
172 /**
173 * Start a batch scrape job for multiple URLs (async).
174 * @param urls URLs to scrape.
175 * @param opts Batch options (scrape options, webhook, concurrency, idempotency key, etc.).
176 * @returns Job id and url.
177 */
106 async startBatchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2]): Promise<BatchScrapeResponse> {178 async startBatchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2]): Promise<BatchScrapeResponse> {
107 return startBatchScrape(this.http, urls, opts);179 return startBatchScrape(this.http, urls, opts);
108 }180 }
181 /**
182 * Get the status and partial data of a batch scrape job.
183 * @param jobId Batch job id.
184 */
109 async getBatchScrapeStatus(jobId: string): Promise<BatchScrapeJob> {185 async getBatchScrapeStatus(jobId: string): Promise<BatchScrapeJob> {
110 return getBatchScrapeStatus(this.http, jobId);186 return getBatchScrapeStatus(this.http, jobId);
111 }187 }
188 /**
189 * Retrieve batch scrape errors and robots.txt blocks.
190 * @param jobId Batch job id.
191 */
112 async getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse> {192 async getBatchScrapeErrors(jobId: string): Promise<CrawlErrorsResponse> {
113 return getBatchScrapeErrors(this.http, jobId);193 return getBatchScrapeErrors(this.http, jobId);
114 }194 }
195 /**
196 * Cancel a batch scrape job.
197 * @param jobId Batch job id.
198 * @returns True if cancelled.
199 */
115 async cancelBatchScrape(jobId: string): Promise<boolean> {200 async cancelBatchScrape(jobId: string): Promise<boolean> {
116 return cancelBatchScrape(this.http, jobId);201 return cancelBatchScrape(this.http, jobId);
117 }202 }
203 /**
204 * Convenience waiter: start a batch scrape and poll until it finishes.
205 * @param urls URLs to scrape.
206 * @param opts Batch options plus waiter controls (pollInterval, timeout seconds).
207 * @returns Final job snapshot.
208 */
118 async batchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2] & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob> {209 async batchScrape(urls: string[], opts?: Parameters<typeof startBatchScrape>[2] & { pollInterval?: number; timeout?: number }): Promise<BatchScrapeJob> {
119 return batchWaiter(this.http, urls, opts);210 return batchWaiter(this.http, urls, opts);
120 }211 }
121 212
122 // Extract213 // Extract
214 /**
215 * Start an extract job (async).
216 * @param args Extraction request (urls, schema or prompt, flags).
217 * @returns Job id or processing state.
218 */
123 async startExtract(args: Parameters<typeof startExtract>[1]): Promise<ExtractResponse> {219 async startExtract(args: Parameters<typeof startExtract>[1]): Promise<ExtractResponse> {
124 return startExtract(this.http, args);220 return startExtract(this.http, args);
125 }221 }
222 /**
223 * Get extract job status/data.
224 * @param jobId Extract job id.
225 */
126 async getExtractStatus(jobId: string): Promise<ExtractResponse> {226 async getExtractStatus(jobId: string): Promise<ExtractResponse> {
127 return getExtractStatus(this.http, jobId);227 return getExtractStatus(this.http, jobId);
128 }228 }
229 /**
230 * Convenience waiter: start an extract and poll until it finishes.
231 * @param args Extraction request plus waiter controls (pollInterval, timeout seconds).
232 * @returns Final extract response.
233 */
129 async extract(args: Parameters<typeof startExtract>[1] & { pollInterval?: number; timeout?: number }): Promise<ExtractResponse> {234 async extract(args: Parameters<typeof startExtract>[1] & { pollInterval?: number; timeout?: number }): Promise<ExtractResponse> {
130 return extractWaiter(this.http, args);235 return extractWaiter(this.http, args);
131 }236 }
132 237
133 // Usage238 // Usage
239 /** Current concurrency usage. */
134 async getConcurrency() {240 async getConcurrency() {
135 return getConcurrency(this.http);241 return getConcurrency(this.http);
136 }242 }
243 /** Current credit usage. */
137 async getCreditUsage() {244 async getCreditUsage() {
138 return getCreditUsage(this.http);245 return getCreditUsage(this.http);
139 }246 }
247 /** Recent token usage. */
140 async getTokenUsage() {248 async getTokenUsage() {
141 return getTokenUsage(this.http);249 return getTokenUsage(this.http);
142 }250 }
143 251
144 // Watcher252 // Watcher
253 /**
254 * Create a watcher for a crawl or batch job. Emits: `document`, `snapshot`, `done`, `error`.
255 * @param jobId Job id.
256 * @param opts Watcher options (kind, pollInterval, timeout seconds).
257 */
145 watcher(jobId: string, opts: WatcherOptions = {}): Watcher {258 watcher(jobId: string, opts: WatcherOptions = {}): Watcher {
146 return new Watcher(this.http, jobId, opts);259 return new Watcher(this.http, jobId, opts);
147 }260 }
apps/js-sdk/firecrawl/src/v2/types.ts+3−3
@@ -179,9 +179,9 @@ export interface SearchResult { · 178 unmodified lines
179}179}
180 180
181export interface SearchData {181export interface SearchData {
182 web?: Array<SearchResult>;182 web?: Array<SearchResult | Document>;
183 news?: Array<SearchResult>;183 news?: Array<SearchResult | Document>;
184 images?: Array<SearchResult>;184 images?: Array<SearchResult | Document>;
185}185}
186 186
187export interface SearchRequest {187export interface SearchRequest {
apps/python-sdk/.env.example+4−1
@@ -1,2 +1,5 @@
1FIRECRAWL_API_KEY=<your api key>1FIRECRAWL_API_KEY=<your api key>
2FIRECRAWL_API_URL=https://api.firecrawl.dev2FIRECRAWL_API_URL=https://api.firecrawl.dev
3
4# TESTING PURPOSE:
5IDMUX_URL=
apps/python-sdk/README.md+44−28
@@ -13,24 +13,25 @@ pip install firecrawl-py · 12 unmodified lines
13## Usage13## Usage
14 14
151. Get an API key from [firecrawl.dev](https://firecrawl.dev)151. Get an API key from [firecrawl.dev](https://firecrawl.dev)
162. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.162. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `Firecrawl` class.
17 17
18Here's an example of how to use the SDK:18Here's an example of how to use the SDK:
19 19
20```python 20```python
21from firecrawl import FirecrawlApp, ScrapeOptions21from firecrawl import Firecrawl
22from firecrawl.types import ScrapeOptions
22 23
23app = FirecrawlApp(api_key="fc-YOUR_API_KEY")24firecrawl = Firecrawl(api_key="fc-YOUR_API_KEY")
24 25
25# Scrape a website:26# Scrape a website (v2):
26data = app.scrape_url(27data = firecrawl.scrape(
27 'https://firecrawl.dev', 28 'https://firecrawl.dev',
28 formats=['markdown', 'html']29 formats=['markdown', 'html']
29)30)
30print(data)31print(data)
31 32
32# Crawl a website:33# Crawl a website (v2 waiter):
33crawl_status = app.crawl_url(34crawl_status = firecrawl.crawl(
34 'https://firecrawl.dev', 35 'https://firecrawl.dev',
35 limit=100, 36 limit=100,
36 scrape_options=ScrapeOptions(formats=['markdown', 'html'])37 scrape_options=ScrapeOptions(formats=['markdown', 'html'])
@@ -40,20 +41,20 @@ print(crawl_status) · 3 unmodified lines
40 41
41### Scraping a URL42### Scraping a URL
42 43
43To scrape a single URL, use the `scrape_url` method. It takes the URL as a parameter and returns the scraped data as a dictionary.44To scrape a single URL, use the `scrape` method. It takes the URL as a parameter and returns a document with the requested formats.
44 45
45```python 46```python
46# Scrape a website:47# Scrape a website (v2):
47scrape_result = app.scrape_url('firecrawl.dev', formats=['markdown', 'html'])48scrape_result = firecrawl.scrape('https://firecrawl.dev', formats=['markdown', 'html'])
48print(scrape_result)49print(scrape_result)
49```50```
50 51
51### Crawling a Website52### Crawling a Website
52 53
53To crawl a website, use the `crawl_url` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.54To crawl a website, use the `crawl` method. It takes the starting URL and optional parameters as arguments. You can control depth, limits, formats, and more.
54 55
55```python 56```python
56crawl_status = app.crawl_url(57crawl_status = firecrawl.crawl(
57 'https://firecrawl.dev', 58 'https://firecrawl.dev',
58 limit=100, 59 limit=100,
59 scrape_options=ScrapeOptions(formats=['markdown', 'html']),60 scrape_options=ScrapeOptions(formats=['markdown', 'html']),
@@ -66,23 +67,23 @@ print(crawl_status) · 6 unmodified lines
66 67
67<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>68<Tip>Looking for async operations? Check out the [Async Class](#async-class) section below.</Tip>
68 69
69To crawl a website asynchronously, use the `crawl_url_async` method. It returns the crawl `ID` which you can use to check the status of the crawl job. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.70To enqueue a crawl asynchronously, use `start_crawl`. It returns the crawl `ID` which you can use to check the status of the crawl job.
70 71
71```python 72```python
72crawl_status = app.async_crawl_url(73crawl_job = firecrawl.start_crawl(
73 'https://firecrawl.dev', 74 'https://firecrawl.dev',
74 limit=100, 75 limit=100,
75 scrape_options=ScrapeOptions(formats=['markdown', 'html']),76 scrape_options=ScrapeOptions(formats=['markdown', 'html']),
76)77)
77print(crawl_status)78print(crawl_job)
78```79```
79 80
80### Checking Crawl Status81### Checking Crawl Status
81 82
82To check the status of a crawl job, use the `check_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.83To check the status of a crawl job, use the `get_crawl_status` method. It takes the job ID as a parameter and returns the current status of the crawl job.
83 84
84```python 85```python
85crawl_status = app.check_crawl_status("<crawl_id>")86crawl_status = firecrawl.get_crawl_status("<crawl_id>")
86print(crawl_status)87print(crawl_status)
87```88```
88 89
@@ -91,17 +92,17 @@ print(crawl_status) · 2 unmodified lines
91To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.92To cancel an asynchronous crawl job, use the `cancel_crawl` method. It takes the job ID of the asynchronous crawl as a parameter and returns the cancellation status.
92 93
93```python 94```python
94cancel_crawl = app.cancel_crawl(id)95cancel_crawl = firecrawl.cancel_crawl(id)
95print(cancel_crawl)96print(cancel_crawl)
96```97```
97 98
98### Map a Website99### Map a Website
99 100
100Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap.101Use `map` to generate a list of URLs from a website. Options let you customize the mapping process, including whether to use the sitemap or include subdomains.
101 102
102```python 103```python
103# Map a website:104# Map a website (v2):
104map_result = app.map_url('https://firecrawl.dev')105map_result = firecrawl.map('https://firecrawl.dev')
105print(map_result)106print(map_result)
106```107```
107 108
@@ -152,20 +153,35 @@ The SDK handles errors returned by the Firecrawl API and raises appropriate exce · 44 unmodified lines
152 153
153## Async Class154## Async Class
154 155
155For async operations, you can use the `AsyncFirecrawlApp` class. Its methods are the same as the `FirecrawlApp` class, but they don't block the main thread.156For async operations, you can use the `AsyncFirecrawl` class. Its methods mirror the `Firecrawl` class, but you `await` them.
156 157
157```python 158```python
158from firecrawl import AsyncFirecrawlApp159from firecrawl import AsyncFirecrawl
159 160
160app = AsyncFirecrawlApp(api_key="YOUR_API_KEY")161firecrawl = AsyncFirecrawl(api_key="YOUR_API_KEY")
161 162
162# Async Scrape163# Async Scrape (v2)
163async def example_scrape():164async def example_scrape():
164 scrape_result = await app.scrape_url(url="https://example.com")165 scrape_result = await firecrawl.scrape(url="https://example.com")
165 print(scrape_result)166 print(scrape_result)
166 167
167# Async Crawl168# Async Crawl (v2)
168async def example_crawl():169async def example_crawl():
169 crawl_result = await app.crawl_url(url="https://example.com")170 crawl_result = await firecrawl.crawl(url="https://example.com")
170 print(crawl_result)171 print(crawl_result)
172```
173
174## v1 compatibility
175
176For legacy code paths, v1 remains available under `firecrawl.v1` with the original method names.
177
178```python
179from firecrawl import Firecrawl
180
181firecrawl = Firecrawl(api_key="YOUR_API_KEY")
182
183# v1 methods (feature‑frozen)
184doc_v1 = firecrawl.v1.scrape_url('https://firecrawl.dev', formats=['markdown', 'html'])
185crawl_v1 = firecrawl.v1.crawl_url('https://firecrawl.dev', limit=100)
186map_v1 = firecrawl.v1.map_url('https://firecrawl.dev')
171```187```
apps/python-sdk/firecrawl/__tests__/e2e/v2/.env.example+5−2
@@ -1,2 +1,5 @@
1API_URL=http://localhost:30021FIRECRAWL_API_KEY=<your api key>
2API_KEY=fc-YOUR_API_KEY2FIRECRAWL_API_URL=https://api.firecrawl.dev
3
4# TESTING PURPOSE:
5IDMUX_URL=
apps/python-sdk/firecrawl/__tests__/e2e/v2/aio/test_aio_map.py+0−2
@@ -17,7 +17,6 @@ if not os.getenv("API_URL"): · 16 unmodified lines
17async def test_async_map_minimal():17async def test_async_map_minimal():
18 client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))18 client = AsyncFirecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
19 resp = await client.v2.map("https://docs.firecrawl.dev")19 resp = await client.v2.map("https://docs.firecrawl.dev")
20 assert resp.success is True
21 assert hasattr(resp, "links") and isinstance(resp.links, list)20 assert hasattr(resp, "links") and isinstance(resp.links, list)
22 if resp.links:21 if resp.links:
23 first = resp.links[0]22 first = resp.links[0]
@@ -36,7 +35,6 @@ async def test_async_map_with_all_params(sitemap): · 12 unmodified lines
36 sitemap=sitemap,35 sitemap=sitemap,
37 timeout=15000,36 timeout=15000,
38 )37 )
39 assert resp.success is True
40 assert hasattr(resp, "links") and isinstance(resp.links, list)38 assert hasattr(resp, "links") and isinstance(resp.links, list)
41 assert len(resp.links) <= 1039 assert len(resp.links) <= 10
42 40
apps/python-sdk/firecrawl/__tests__/e2e/v2/conftest.py+73−0
@@ -0,0 +1,73 @@
1import os
2import json
3import pytest
4import requests
5from dotenv import load_dotenv
6
7load_dotenv()
8
9def _idmux(identity_request: dict) -> dict:
10 idmux_url = os.getenv("IDMUX_URL")
11 if not idmux_url:
12 raise EnvironmentError("IDMUX_URL is not set. E2E tests must use idmux for credentials.")
13 run_number = int(os.getenv("GITHUB_RUN_NUMBER") or 0)
14 payload = {
15 "refName": os.getenv("GITHUB_REF_NAME") or "local",
16 "runNumber": run_number,
17 "concurrency": identity_request.get("concurrency", 100),
18 **identity_request,
19 }
20 resp = requests.post(idmux_url + "/", json=payload)
21 resp.raise_for_status()
22 return resp.json()
23
24@pytest.fixture(scope="session")
25def api_url():
26 # Prefer TEST_URL, then FIRECRAWL_API_URL (for parity with JS), then legacy API_URL
27 return (
28 os.getenv("TEST_URL")
29 or os.getenv("FIRECRAWL_API_URL")
30 or os.getenv("API_URL")
31 or "https://api.firecrawl.dev"
32 )
33
34# Resolve identity and export environment at import time so tests that read env at module import succeed
35_IDENTITY = None
36_API_URL = (
37 os.getenv("TEST_URL")
38 or os.getenv("FIRECRAWL_API_URL")
39 or os.getenv("API_URL")
40 or "https://api.firecrawl.dev"
41)
42
43_IDMUX_URL = os.getenv("IDMUX_URL")
44if _IDMUX_URL:
45 run_name = os.getenv("PYTEST_RUN_NAME") or "py-e2e"
46 # If IDMUX_URL is set, idmux MUST succeed; do not silently fall back
47 _IDENTITY = _idmux({"name": run_name})
48 os.environ["API_KEY"] = _IDENTITY.get("apiKey", "")
49 os.environ["API_URL"] = _API_URL
50
51@pytest.fixture(scope="session")
52def api_identity():
53 return _IDENTITY or {"apiKey": os.getenv("API_KEY") or "", "teamId": os.getenv("TEST_TEAM_ID") or os.getenv("TEAM_ID") or ""}
54
55@pytest.fixture(autouse=True)
56def _inject_client(request, api_identity, api_url):
57 # For class-based tests that rely on self.client, inject a client if missing
58 inst = getattr(request, "instance", None)
59 if inst is not None and not hasattr(inst, "client"):
60 try:
61 from firecrawl import Firecrawl
62 inst.client = Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url)
63 except Exception:
64 pass
65 # For function-based modules that expect a module-level `firecrawl` symbol
66 mod = getattr(request, "module", None)
67 if mod is not None and not hasattr(mod, "firecrawl"):
68 try:
69 from firecrawl import Firecrawl
70 setattr(mod, "firecrawl", Firecrawl(api_key=api_identity.get("apiKey", ""), api_url=api_url))
71 except Exception:
72 pass
73
apps/python-sdk/firecrawl/__tests__/e2e/v2/test_map.py+1−4
@@ -24,7 +24,6 @@ class TestMapE2E: · 23 unmodified lines
24 def test_map_minimal_request(self):24 def test_map_minimal_request(self):
25 resp = self.client.map("https://docs.firecrawl.dev")25 resp = self.client.map("https://docs.firecrawl.dev")
26 26
27 assert hasattr(resp, "success") and resp.success is True
28 assert hasattr(resp, "links") and resp.links is not None27 assert hasattr(resp, "links") and resp.links is not None
29 assert isinstance(resp.links, list)28 assert isinstance(resp.links, list)
30 29
@@ -46,13 +45,11 @@ class TestMapE2E: · 15 unmodified lines
46 resp = self.client.map(45 resp = self.client.map(
47 "https://docs.firecrawl.dev",46 "https://docs.firecrawl.dev",
48 search="docs",47 search="docs",
49 include_subdomains=True,48 include_subdomains=True, limit=10,
50 limit=10,
51 sitemap=sitemap,49 sitemap=sitemap,
52 timeout=15000,50 timeout=15000,
53 )51 )
54 52
55 assert hasattr(resp, "success") and resp.success is True
56 assert hasattr(resp, "links") and isinstance(resp.links, list)53 assert hasattr(resp, "links") and isinstance(resp.links, list)
57 54
58 # Limit should be respected (server-side)55 # Limit should be respected (server-side)
apps/python-sdk/firecrawl/__tests__/e2e/v2/test_search.py+0−6
@@ -5,12 +5,6 @@ from firecrawl.types import SearchData, SearchResult, Document, ScrapeFormats, S · 4 unmodified lines
5 5
6load_dotenv()6load_dotenv()
7 7
8if not os.getenv("API_KEY"):
9 raise ValueError("API_KEY is not set")
10
11if not os.getenv("API_URL"):
12 raise ValueError("API_URL is not set")
13
14firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))8firecrawl = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
15 9
16def _collect_texts(entries):10def _collect_texts(entries):
apps/python-sdk/firecrawl/__tests__/e2e/v2/test_usage.py+1−6
@@ -4,15 +4,10 @@ from firecrawl import Firecrawl · 3 unmodified lines
4 4
5load_dotenv()5load_dotenv()
6 6
7if not os.getenv("API_KEY"):
8 raise ValueError("API_KEY is not set")
9
10if not os.getenv("API_URL"):
11 raise ValueError("API_URL is not set")
12
13 7
14class TestUsageE2E:8class TestUsageE2E:
15 def setup_method(self):9 def setup_method(self):
10 # Environment is exported by conftest at import time
16 self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))11 self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
17 12
18 def test_get_concurrency(self):13 def test_get_concurrency(self):
apps/python-sdk/firecrawl/__tests__/e2e/v2/test_watcher.py+1−0
@@ -14,6 +14,7 @@ if not os.getenv("API_URL"): · 13 unmodified lines
14 14
15class TestWatcherE2E:15class TestWatcherE2E:
16 def setup_method(self):16 def setup_method(self):
17 from firecrawl import Firecrawl
17 self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))18 self.client = Firecrawl(api_key=os.getenv("API_KEY"), api_url=os.getenv("API_URL"))
18 19
19 def test_crawl_watcher(self):20 def test_crawl_watcher(self):
apps/python-sdk/firecrawl/client.py+39−17
@@ -113,20 +113,18 @@ class AsyncV2Proxy: · 112 unmodified lines
113 113
114class Firecrawl:114class Firecrawl:
115 """115 """
116 Unified Firecrawl client with version-specific access patterns.116 Unified Firecrawl client (v2 by default, v1 under ``.v1``).
117 117
118 This class provides a unified interface that allows users to access118 Provides a single entrypoint that exposes the latest API directly while
119 different API versions through version-specific attributes (v1, v2)119 keeping a feature-frozen v1 available for incremental migration.
120 while defaulting to v2 for direct method calls.
121 """120 """
122 121
123 def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):122 def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
124 """123 """Initialize the unified client.
125 Initialize the unified Firecrawl client.124
126
127 Args:125 Args:
128 api_key: Your Firecrawl API key126 api_key: Firecrawl API key (or set ``FIRECRAWL_API_KEY``)
129 api_url: The Firecrawl API URL (defaults to production)127 api_url: Base API URL (defaults to production)
130 """128 """
131 self.api_key = api_key129 self.api_key = api_key
132 self.api_url = api_url130 self.api_url = api_url
@@ -164,13 +162,7 @@ class Firecrawl: · 31 unmodified lines
164 self.watcher = self._v2_client.watcher162 self.watcher = self._v2_client.watcher
165 163
166class AsyncFirecrawl:164class AsyncFirecrawl:
167 """165 """Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""
168 Unified Firecrawl client with version-specific access patterns.
169
170 This class provides a unified interface that allows users to access
171 different API versions through version-specific attributes (v1, v2)
172 while defaulting to v2 for direct method calls.
173 """
174 166
175 def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):167 def __init__(self, api_key: str = None, api_url: str = "https://api.firecrawl.dev"):
176 self.api_key = api_key168 self.api_key = api_key
@@ -185,6 +177,36 @@ class AsyncFirecrawl: · 8 unmodified lines
185 self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None177 self.v1 = AsyncV1Proxy(self._v1_client) if self._v1_client else None
186 self.v2 = AsyncV2Proxy(self._v2_client)178 self.v2 = AsyncV2Proxy(self._v2_client)
187 179
180 # Expose v2 async surface directly on the top-level client for ergonomic access
181 # Keep method names aligned with the sync client
182 self.scrape = self._v2_client.scrape
183 self.search = self._v2_client.search
184 self.map = self._v2_client.map
185
186 self.start_crawl = self._v2_client.start_crawl
187 self.get_crawl_status = self._v2_client.get_crawl_status
188 self.cancel_crawl = self._v2_client.cancel_crawl
189 self.crawl = self._v2_client.crawl
190 self.get_crawl_errors = self._v2_client.get_crawl_errors
191 self.active_crawls = self._v2_client.active_crawls
192 self.crawl_params_preview = self._v2_client.crawl_params_preview
193
194 self.start_batch_scrape = self._v2_client.start_batch_scrape
195 self.get_batch_scrape_status = self._v2_client.get_batch_scrape_status
196 self.cancel_batch_scrape = self._v2_client.cancel_batch_scrape
197 self.batch_scrape = self._v2_client.batch_scrape
198 self.get_batch_scrape_errors = self._v2_client.get_batch_scrape_errors
199
200 self.start_extract = self._v2_client.start_extract
201 self.get_extract_status = self._v2_client.get_extract_status
202 self.extract = self._v2_client.extract
203
204 self.get_concurrency = self._v2_client.get_concurrency
205 self.get_credit_usage = self._v2_client.get_credit_usage
206 self.get_token_usage = self._v2_client.get_token_usage
207
208 self.watcher = self._v2_client.watcher
209
188# Export Firecrawl as an alias for FirecrawlApp210# Export Firecrawl as an alias for FirecrawlApp
189FirecrawlApp = Firecrawl211FirecrawlApp = Firecrawl
190AsyncFirecrawlApp = AsyncFirecrawl212AsyncFirecrawlApp = AsyncFirecrawl
apps/python-sdk/firecrawl/firecrawl.backup.pyrenamed from apps/python-sdk/firecrawl/DELETE_firecrawl.py+0−0

File renamed without changes.

apps/python-sdk/firecrawl/v2/client.py+128−14
@@ -408,8 +408,18 @@ class FirecrawlClient: · 407 unmodified lines
408 sitemap: Optional[Literal["only", "include", "skip"]] = None,408 sitemap: Optional[Literal["only", "include", "skip"]] = None,
409 timeout: Optional[int] = None,409 timeout: Optional[int] = None,
410 ) -> MapData:410 ) -> MapData:
411 """411 """Map a URL and return discovered links.
412 Map a URL and return discovered links (with optional titles/descriptions).412
413 Args:
414 url: Root URL to explore
415 search: Optional substring filter for discovered links
416 include_subdomains: Whether to include subdomains
417 limit: Maximum number of links to return
418 sitemap: Sitemap usage mode ("only" | "include" | "skip")
419 timeout: Request timeout in milliseconds
420
421 Returns:
422 MapData containing the discovered links
413 """423 """
414 options = MapOptions(424 options = MapOptions(
415 search=search,425 search=search,
@@ -434,8 +444,14 @@ class FirecrawlClient: · 18 unmodified lines
434 return crawl_module.cancel_crawl(self.http_client, crawl_id)444 return crawl_module.cancel_crawl(self.http_client, crawl_id)
435 445
436 def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:446 def crawl_params_preview(self, url: str, prompt: str) -> CrawlParamsData:
437 """447 """Derive crawl parameters from natural-language prompt.
438 Get crawl parameters from LLM based on URL and prompt.448
449 Args:
450 url: Root URL
451 prompt: Instruction describing how to crawl
452
453 Returns:
454 CrawlParamsData with normalized crawl configuration
439 """455 """
440 request = CrawlParamsRequest(url=url, prompt=prompt)456 request = CrawlParamsRequest(url=url, prompt=prompt)
441 return crawl_module.crawl_params_preview(self.http_client, request)457 return crawl_module.crawl_params_preview(self.http_client, request)
@@ -453,8 +469,21 @@ class FirecrawlClient: · 11 unmodified lines
453 scrape_options: Optional['ScrapeOptions'] = None,469 scrape_options: Optional['ScrapeOptions'] = None,
454 ignore_invalid_urls: Optional[bool] = None,470 ignore_invalid_urls: Optional[bool] = None,
455 ):471 ):
456 """472 """Start an extract job (non-blocking).
457 Start an extract job (non-blocking). Use get_extract_status to poll.473
474 Args:
475 urls: URLs to extract from (optional)
476 prompt: Natural-language instruction for extraction
477 schema: Target JSON schema for the output
478 system_prompt: Optional system instruction
479 allow_external_links: Allow hyperlinks in output
480 enable_web_search: Whether to augment with web search
481 show_sources: Include per-field/source mapping when available
482 scrape_options: Scrape options applied prior to extraction
483 ignore_invalid_urls: Skip invalid URLs instead of failing
484
485 Returns:
486 Response payload with job id/status (poll with get_extract_status)
458 """487 """
459 return extract_module.start_extract(488 return extract_module.start_extract(
460 self.http_client,489 self.http_client,
@@ -484,8 +513,23 @@ class FirecrawlClient: · 23 unmodified lines
484 poll_interval: int = 2,513 poll_interval: int = 2,
485 timeout: Optional[int] = None,514 timeout: Optional[int] = None,
486 ):515 ):
487 """516 """Extract structured data and wait until completion.
488 Extract structured data using the v2 client surface, waiting until completion.517
518 Args:
519 urls: URLs to extract from (optional)
520 prompt: Natural-language instruction for extraction
521 schema: Target JSON schema for the output
522 system_prompt: Optional system instruction
523 allow_external_links: Allow hyperlinks in output
524 enable_web_search: Whether to augment with web search
525 show_sources: Include per-field/source mapping when available
526 scrape_options: Scrape options applied prior to extraction
527 ignore_invalid_urls: Skip invalid URLs instead of failing
528 poll_interval: Seconds between status checks
529 timeout: Maximum seconds to wait (None for no timeout)
530
531 Returns:
532 Final extract response when completed
489 """533 """
490 return extract_module.extract(534 return extract_module.extract(
491 self.http_client,535 self.http_client,
@@ -533,8 +577,39 @@ class FirecrawlClient: · 41 unmodified lines
533 integration: Optional[str] = None,577 integration: Optional[str] = None,
534 idempotency_key: Optional[str] = None,578 idempotency_key: Optional[str] = None,
535 ):579 ):
536 """580 """Start a batch scrape job over multiple URLs (non-blocking).
537 Start a batch scrape job over multiple URLs.581
582 Args:
583 urls: List of URLs to scrape
584 formats: Output formats to collect per URL
585 headers: HTTP headers
586 include_tags: HTML tags to include
587 exclude_tags: HTML tags to exclude
588 only_main_content: Restrict scraping to main content
589 timeout: Per-request timeout in milliseconds
590 wait_for: Wait condition in milliseconds
591 mobile: Emulate mobile viewport
592 parsers: Parser list (e.g., ["pdf"])
593 actions: Browser actions to perform
594 location: Location settings
595 skip_tls_verification: Skip TLS verification
596 remove_base64_images: Remove base64 images from output
597 fast_mode: Prefer faster scraping modes
598 use_mock: Use a mock data source (internal/testing)
599 block_ads: Block ads during scraping
600 proxy: Proxy setting
601 max_age: Cache max age
602 store_in_cache: Whether to store results in cache
603 webhook: Webhook configuration
604 append_to_id: Append to an existing batch job
605 ignore_invalid_urls: Skip invalid URLs without failing
606 max_concurrency: Max concurrent scrapes
607 zero_data_retention: Delete data after 24 hours
608 integration: Integration tag/name
609 idempotency_key: Header used to deduplicate starts
610
611 Returns:
612 Response payload with job id (poll with get_batch_scrape_status)
538 """613 """
539 options = ScrapeOptions(614 options = ScrapeOptions(
540 **{k: v for k, v in dict(615 **{k: v for k, v in dict(
@@ -574,18 +649,47 @@ class FirecrawlClient: · 33 unmodified lines
574 )649 )
575 650
576 def get_batch_scrape_status(self, job_id: str):651 def get_batch_scrape_status(self, job_id: str):
577 """Get current status and any scraped data for a batch scrape job."""652 """Get current status and any scraped data for a batch job.
653
654 Args:
655 job_id: Batch job ID
656
657 Returns:
658 Status payload including counts and partial data
659 """
578 return batch_module.get_batch_scrape_status(self.http_client, job_id)660 return batch_module.get_batch_scrape_status(self.http_client, job_id)
579 661
580 def cancel_batch_scrape(self, job_id: str) -> bool:662 def cancel_batch_scrape(self, job_id: str) -> bool:
581 """Cancel a running batch scrape job."""663 """Cancel a running batch scrape job.
664
665 Args:
666 job_id: Batch job ID
667
668 Returns:
669 True if the job was cancelled
670 """
582 return batch_module.cancel_batch_scrape(self.http_client, job_id)671 return batch_module.cancel_batch_scrape(self.http_client, job_id)
583 672
584 def get_batch_scrape_errors(self, job_id: str):673 def get_batch_scrape_errors(self, job_id: str):
585 """Retrieve error details for a batch scrape job."""674 """Retrieve error details for a batch scrape job.
675
676 Args:
677 job_id: Batch job ID
678
679 Returns:
680 Errors and robots-blocked URLs for the job
681 """
586 return batch_methods.get_batch_scrape_errors(self.http_client, job_id)682 return batch_methods.get_batch_scrape_errors(self.http_client, job_id)
587 683
588 def get_extract_status(self, job_id: str):684 def get_extract_status(self, job_id: str):
685 """Get the current status (and data if completed) of an extract job.
686
687 Args:
688 job_id: Extract job ID
689
690 Returns:
691 Extract response payload with status and optional data
692 """
589 return extract_module.get_extract_status(self.http_client, job_id)693 return extract_module.get_extract_status(self.http_client, job_id)
590 694
591 def get_concurrency(self):695 def get_concurrency(self):
@@ -608,7 +712,17 @@ class FirecrawlClient: · 16 unmodified lines
608 poll_interval: int = 2,712 poll_interval: int = 2,
609 timeout: Optional[int] = None,713 timeout: Optional[int] = None,
610 ) -> Watcher:714 ) -> Watcher:
611 """Create a polling-based watcher for crawl or batch jobs."""715 """Create a watcher for crawl or batch jobs.
716
717 Args:
718 job_id: Job ID to watch
719 kind: Job kind ("crawl" or "batch")
720 poll_interval: Seconds between status checks
721 timeout: Maximum seconds to watch (None for no timeout)
722
723 Returns:
724 Watcher instance
725 """
612 return Watcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout)726 return Watcher(self, job_id, kind=kind, poll_interval=poll_interval, timeout=timeout)
613 727
614 def batch_scrape(728 def batch_scrape(
apps/python-sdk/firecrawl/v2/methods/aio/crawl.py+1−2
@@ -160,8 +160,7 @@ async def get_active_crawls(client: AsyncHttpClient) -> ActiveCrawlsResponse: · 159 unmodified lines
160 body = response.json()160 body = response.json()
161 if not body.get("success"):161 if not body.get("success"):
162 raise Exception(body.get("error", "Unknown error occurred"))162 raise Exception(body.get("error", "Unknown error occurred"))
163 data = body.get("data", {})163 crawls_in = body.get("crawls", [])
164 crawls_in = data.get("crawls", [])
165 normalized = []164 normalized = []
166 for c in crawls_in:165 for c in crawls_in:
167 if isinstance(c, dict):166 if isinstance(c, dict):
apps/python-sdk/firecrawl/v2/methods/aio/usage.py+2−2
@@ -18,7 +18,7 @@ async def get_concurrency(client: AsyncHttpClient) -> ConcurrencyCheck: · 17 unmodified lines
18 18
19 19
20async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage:20async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage:
21 resp = await client.get("/v2/team/credit-usage")21 resp = await client.get("/v2/credit-usage")
22 if resp.status_code >= 400:22 if resp.status_code >= 400:
23 handle_response_error(resp, "get credit usage")23 handle_response_error(resp, "get credit usage")
24 body = resp.json()24 body = resp.json()
@@ -29,7 +29,7 @@ async def get_credit_usage(client: AsyncHttpClient) -> CreditUsage: · 4 unmodified lines
29 29
30 30
31async def get_token_usage(client: AsyncHttpClient) -> TokenUsage:31async def get_token_usage(client: AsyncHttpClient) -> TokenUsage:
32 resp = await client.get("/v2/team/token-usage")32 resp = await client.get("/v2/token-usage")
33 if resp.status_code >= 400:33 if resp.status_code >= 400:
34 handle_response_error(resp, "get token usage")34 handle_response_error(resp, "get token usage")
35 body = resp.json()35 body = resp.json()