History

Nick:

Nicolas committed 1 year ago

Commit b98e2891 parent a185c054 files · +47 −2Browse files

Unified · Split

4 files
  1. apps/api/src/controllers/v1/extract.ts +25 −0
  2. apps/api/src/lib/extract/team-id-sync.ts +19 −0
  3. apps/js-sdk/firecrawl/src/index.ts +1 −1
  4. apps/python-sdk/firecrawl/firecrawl.py +2 −1
apps/api/src/controllers/v1/extract.ts+25−0
@@ -8,7 +8,28 @@ import { · 7 unmodified lines
8import { getExtractQueue } from "../../services/queue-service";8import { getExtractQueue } from "../../services/queue-service";
9import * as Sentry from "@sentry/node";9import * as Sentry from "@sentry/node";
10import { saveExtract } from "../../lib/extract/extract-redis";10import { saveExtract } from "../../lib/extract/extract-redis";
11import { getTeamIdSyncB } from "../../lib/extract/team-id-sync";
12import { performExtraction } from "../../lib/extract/extraction-service";
11 13
14export async function oldExtract(req: RequestWithAuth<{}, ExtractResponse, ExtractRequest>, res: Response<ExtractResponse>, extractId: string){
15 // Means that are in the non-queue system
16 // TODO: Remove this once all teams have transitioned to the new system
17 try {
18 const result = await performExtraction(extractId, {
19 request: req.body,
20 teamId: req.auth.team_id,
21 plan: req.auth.plan ?? "free",
22 subId: req.acuc?.sub_id ?? undefined,
23 });
24
25 return res.status(200).json(result);
26 } catch (error) {
27 return res.status(500).json({
28 success: false,
29 error: "Internal server error",
30 });
31 }
32 }
12/**33/**
13 * Extracts data from the provided URLs based on the request parameters.34 * Extracts data from the provided URLs based on the request parameters.
14 * Currently in beta.35 * Currently in beta.
@@ -32,6 +53,10 @@ export async function extractController( · 17 unmodified lines
32 extractId,53 extractId,
33 };54 };
34 55
56 if(await getTeamIdSyncB(req.auth.team_id) && req.body.origin !== "api-sdk") {
57 return await oldExtract(req, res, extractId);
58 }
59
35 await saveExtract(extractId, {60 await saveExtract(extractId, {
36 id: extractId,61 id: extractId,
37 team_id: req.auth.team_id,62 team_id: req.auth.team_id,
apps/api/src/lib/extract/team-id-sync.ts+19−0
@@ -0,0 +1,19 @@
1import { supabase_service } from "../../services/supabase";
2import { logger } from "../logger";
3
4export async function getTeamIdSyncB(teamId: string) {
5 try {
6 const { data, error } = await supabase_service
7 .from("eb-sync")
8 .select("team_id")
9 .eq("team_id", teamId)
10 .limit(1);
11 if (error) {
12 throw new Error("Error getting team id (sync b)");
13 }
14 return data[0] ?? null;
15 } catch (error) {
16 logger.error("Error getting team id (sync b)", error);
17 return null;
18 }
19}
apps/js-sdk/firecrawl/src/index.ts+1−1
@@ -884,7 +884,7 @@ export default class FirecrawlApp { · 883 unmodified lines
884 try {884 try {
885 const response: AxiosResponse = await this.postRequest(885 const response: AxiosResponse = await this.postRequest(
886 this.apiUrl + `/v1/extract`,886 this.apiUrl + `/v1/extract`,
887 { ...jsonData, schema: jsonSchema },887 { ...jsonData, schema: jsonSchema, origin: "api-sdk" },
888 headers888 headers
889 );889 );
890 890
apps/python-sdk/firecrawl/firecrawl.py+2−1
@@ -538,7 +538,8 @@ class FirecrawlApp: · 537 unmodified lines
538 request_data = {538 request_data = {
539 **jsonData,539 **jsonData,
540 'allowExternalLinks': params.get('allow_external_links', False),540 'allowExternalLinks': params.get('allow_external_links', False),
541 'schema': schema541 'schema': schema,
542 'origin': 'api-sdk'
542 }543 }
543 544
544 try:545 try:

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t