{ "name": "firecrawl-scraper-js", "version": "1.0.0", "description": "", "main": "index.js", "scripts": { "start": "tsc-watch --onSuccess \"node dist/src/index.js\"", "start:production": "tsc && node dist/src/index.js", "start:production:nobuild": "node dist/src/index.js", "flyio": "node dist/src/index.js", "start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"", "build": "tsc", "build:nosentry": "tsc", "test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'", "test:local-no-auth": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_withAuth/*'", "test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'", "test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'", "test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/**/*.test.ts", "harness": "tsx src/harness.ts", "workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"", "worker:production": "node dist/src/services/queue-worker.js", "index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"", "index-worker:production": "node dist/src/services/indexing/index-worker.js", "mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest", "mongo-docker-console": "docker exec -it mongodb mongosh", "run-example": "npx ts-node src/example.ts", "deploy:fly": "flyctl deploy --build-secret SENTRY_AUTH_TOKEN=$(dotenv -p SENTRY_AUTH_TOKEN) --depot=false", "deploy:fly:staging": "fly deploy -c fly.staging.toml --depot=false", "sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist", "prepare": "cd ../.. && husky ./apps/api/.husky" }, "author": "", "license": "ISC", "devDependencies": { "@jest/globals": "^29.7.0", "@tsconfig/recommended": "^1.0.3", "@types/body-parser": "^1.19.2", "@types/cors": "^2.8.13", "@types/escape-html": "^1.0.4", "@types/express": "^4.17.21", "@types/express-ws": "^3.0.5", "@types/jest": "^29.5.12", "@types/lodash": "^4.17.14", "@types/node": "^20.14.1", "@types/pdf-parse": "^1.1.4", "@types/supertest": "^6.0.2", "@types/tough-cookie": "^4.0.5", "husky": "^9.1.7", "jest": "^29.6.3", "jest-fetch-mock": "^3.0.3", "lint-staged": "^16.1.6", "supabase": "^2.39.2", "supertest": "^6.3.3", "ts-jest": "^29.1.1", "ts-node": "^10.9.1", "tsc-watch": "^7.1.1", "tsx": "^4.20.3", "typescript": "^5.8.3" }, "dependencies": { "@ai-sdk/anthropic": "^1.2.4", "@ai-sdk/deepinfra": "^0.2.4", "@ai-sdk/fireworks": "^0.2.4", "@ai-sdk/google": "^1.2.3", "@ai-sdk/google-vertex": "^2.2.15", "@ai-sdk/groq": "^1.2.1", "@ai-sdk/openai": "^1.3.12", "@anthropic-ai/sdk": "^0.24.3", "@apidevtools/json-schema-ref-parser": "^11.7.3", "@brillout/import": "^0.2.2", "@bull-board/api": "^6.11.2", "@bull-board/express": "^6.11.2", "@devil7softwares/pos": "^1.0.2", "@dqbd/tiktoken": "^1.0.17", "@google-cloud/storage": "^7.16.0", "@nangohq/node": "^0.40.8", "@openrouter/ai-sdk-provider": "^0.4.5", "@opentelemetry/auto-instrumentations-node": "^0.62.0", "@opentelemetry/core": "^2.0.1", "@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0", "@opentelemetry/resources": "^2.0.1", "@opentelemetry/sdk-node": "^0.203.0", "@opentelemetry/sdk-trace-node": "^2.0.1", "@opentelemetry/semantic-conventions": "^1.36.0", "@sentry/cli": "^2.33.1", "@sentry/node": "^9.40.0", "@sentry/profiling-node": "^9.40.0", "@supabase/supabase-js": "^2.52.0", "@types/ws": "^8.5.12", "ai": "^4.3.4", "ajv": "^8.16.0", "async": "^3.2.5", "async-mutex": "^0.5.0", "axios": "^1.11.0", "axios-retry": "^4.5.0", "body-parser": "^1.20.3", "bottleneck": "^2.19.5", "bullmq": "^5.56.7", "bullmq-otel": "^1.0.1", "cacheable-lookup": "^6.1.0", "cheerio": "^1.0.0-rc.12", "cohere": "^1.1.1", "cohere-ai": "^7.14.0", "cors": "^2.8.5", "cron-parser": "^4.9.0", "date-fns": "^3.6.0", "dotenv": "^16.3.1", "dotenv-cli": "^7.4.2", "escape-html": "^1.0.3", "express": "^4.21.2", "express-rate-limit": "^7.3.1", "express-ws": "^5.0.2", "geoip-country": "^5.0.202508192342", "git-diff": "^2.0.6", "glob": "^10.4.2", "gpt3-tokenizer": "^1.1.5", "http-cookie-agent": "^7.0.1", "ioredis": "^5.6.1", "ipaddr.js": "^2.2.0", "joplin-turndown-plugin-gfm": "^1.0.12", "jsdom": "^26.0.0", "json-schema-to-zod": "^2.3.0", "keyword-extractor": "^0.0.28", "koffi": "^2.9.0", "langfuse": "^3.38.4", "langfuse-vercel": "^3.38.4", "languagedetect": "^2.0.0", "lodash": "^4.17.21", "logsnag": "^1.0.0", "luxon": "^3.4.3", "mammoth": "^1.7.2", "marked": "^14.1.2", "md5": "^2.3.0", "moment": "^2.29.4", "ollama-ai-provider": "^1.2.0", "openai": "^5.12.0", "parse-diff": "^0.11.1", "pdf-parse": "^1.1.1", "pos": "^0.4.2", "posthog-node": "^4.0.1", "prettier": "^3.6.2", "psl": "^1.15.0", "rate-limiter-flexible": "2.4.2", "redlock": "5.0.0-beta.2", "resend": "^3.4.0", "robots-parser": "^3.0.1", "stripe": "^16.1.0", "systeminformation": "^5.22.11", "tldts": "^6.1.75", "tough-cookie": "^4.1.4", "turndown": "^7.1.3", "turndown-plugin-gfm": "^1.0.2", "undici": "^7.10.0", "unstructured-client": "^0.11.3", "uuid": "^10.0.0", "winston": "^3.14.2", "winston-transport": "^4.8.0", "wordpos": "^2.1.0", "ws": "^8.18.0", "x402-express": "^0.5.3", "xml2js": "^0.6.2", "zod": "^3.24.2" }, "nodemonConfig": { "ignore": [ "*.docx", "*.json", "temp" ] }, "pnpm": { "onlyBuiltDependencies": [ "@sentry-internal/node-cpu-profiler", "supabase" ], "overrides": { "brace-expansion@>=2.0.0 <=2.0.1": "2.0.2" } }, "lint-staged": { "*.{js,jsx,ts,tsx,json,css,md}": "prettier --write" }}
{
"name": "firecrawl-scraper-js",
"version": "1.0.0",
"description": "",
"main": "index.js",
"scripts": {
"start": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"start:production": "tsc && node dist/src/index.js",
"start:production:nobuild": "node dist/src/index.js",
"flyio": "node dist/src/index.js",
"start:dev": "tsc-watch --onSuccess \"node dist/src/index.js\"",
"build": "tsc",
"build:nosentry": "tsc",
"test": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_noAuth/*'",
"test:local-no-auth": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='src/__tests__/e2e_withAuth/*'",
"test:full": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_withAuth)'",
"test:prod": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false --testPathIgnorePatterns='(src/__tests__/e2e_noAuth|src/__tests__/e2e_full_withAuth|src/scraper/scrapeURL)'",
"test:snips": "npx jest --detectOpenHandles --forceExit --openHandlesTimeout=120000 --watchAll=false src/__tests__/snips/**/*.test.ts",
"harness": "tsx src/harness.ts",
"workers": "tsc-watch --onSuccess \"node dist/src/services/queue-worker.js\"",
"worker:production": "node dist/src/services/queue-worker.js",
"index-worker": "tsc-watch --onSuccess \"node dist/src/services/indexing/index-worker.js\"",
"index-worker:production": "node dist/src/services/indexing/index-worker.js",
"mongo-docker": "docker run -d -p 2717:27017 -v ./mongo-data:/data/db --name mongodb mongo:latest",
"mongo-docker-console": "docker exec -it mongodb mongosh",
"run-example": "npx ts-node src/example.ts",
"deploy:fly": "flyctl deploy --build-secret SENTRY_AUTH_TOKEN=$(dotenv -p SENTRY_AUTH_TOKEN) --depot=false",
"deploy:fly:staging": "fly deploy -c fly.staging.toml --depot=false",
"sentry:sourcemaps": "sentry-cli sourcemaps inject --org caleb-peffer --project firecrawl-scraper-js ./dist && sentry-cli sourcemaps upload --org caleb-peffer --project firecrawl-scraper-js ./dist",
"prepare": "cd ../.. && husky ./apps/api/.husky"
},
"author": "",
"license": "ISC",
"devDependencies": {
"@jest/globals": "^29.7.0",
"@tsconfig/recommended": "^1.0.3",
"@types/body-parser": "^1.19.2",
"@types/cors": "^2.8.13",
"@types/escape-html": "^1.0.4",
"@types/express": "^4.17.21",
"@types/express-ws": "^3.0.5",
"@types/jest": "^29.5.12",
"@types/lodash": "^4.17.14",
"@types/node": "^20.14.1",
"@types/pdf-parse": "^1.1.4",
"@types/supertest": "^6.0.2",
"@types/tough-cookie": "^4.0.5",
"husky": "^9.1.7",
"jest": "^29.6.3",
"jest-fetch-mock": "^3.0.3",
"lint-staged": "^16.1.6",
"supabase": "^2.39.2",
"supertest": "^6.3.3",
"ts-jest": "^29.1.1",
"ts-node": "^10.9.1",
"tsc-watch": "^7.1.1",
"tsx": "^4.20.3",
"typescript": "^5.8.3"
},
"dependencies": {
"@ai-sdk/anthropic": "^1.2.4",
"@ai-sdk/deepinfra": "^0.2.4",
"@ai-sdk/fireworks": "^0.2.4",
"@ai-sdk/google": "^1.2.3",
"@ai-sdk/google-vertex": "^2.2.15",
"@ai-sdk/groq": "^1.2.1",
"@ai-sdk/openai": "^1.3.12",
"@anthropic-ai/sdk": "^0.24.3",
"@apidevtools/json-schema-ref-parser": "^11.7.3",
"@brillout/import": "^0.2.2",
"@bull-board/api": "^6.11.2",
"@bull-board/express": "^6.11.2",
"@devil7softwares/pos": "^1.0.2",
"@dqbd/tiktoken": "^1.0.17",
"@google-cloud/storage": "^7.16.0",
"@nangohq/node": "^0.40.8",
"@openrouter/ai-sdk-provider": "^0.4.5",
"@opentelemetry/auto-instrumentations-node": "^0.62.0",
"@opentelemetry/core": "^2.0.1",
"@opentelemetry/exporter-trace-otlp-grpc": "^0.203.0",
"@opentelemetry/resources": "^2.0.1",
"@opentelemetry/sdk-node": "^0.203.0",
"@opentelemetry/sdk-trace-node": "^2.0.1",
"@opentelemetry/semantic-conventions": "^1.36.0",
"@sentry/cli": "^2.33.1",
"@sentry/node": "^9.40.0",
"@sentry/profiling-node": "^9.40.0",
"@supabase/supabase-js": "^2.52.0",
"@types/ws": "^8.5.12",
"ai": "^4.3.4",
"ajv": "^8.16.0",
"async": "^3.2.5",
"async-mutex": "^0.5.0",
"axios": "^1.11.0",
"axios-retry": "^4.5.0",
"body-parser": "^1.20.3",
"bottleneck": "^2.19.5",
"bullmq": "^5.56.7",
"bullmq-otel": "^1.0.1",
"cacheable-lookup": "^6.1.0",
"cheerio": "^1.0.0-rc.12",
"cohere": "^1.1.1",
"cohere-ai": "^7.14.0",
"cors": "^2.8.5",
"cron-parser": "^4.9.0",
"date-fns": "^3.6.0",
"dotenv": "^16.3.1",
"dotenv-cli": "^7.4.2",
"escape-html": "^1.0.3",
"express": "^4.21.2",
"express-rate-limit": "^7.3.1",
"express-ws": "^5.0.2",
"geoip-country": "^5.0.202508192342",
"git-diff": "^2.0.6",
"glob": "^10.4.2",
"gpt3-tokenizer": "^1.1.5",
"http-cookie-agent": "^7.0.1",
"ioredis": "^5.6.1",
"ipaddr.js": "^2.2.0",
"joplin-turndown-plugin-gfm": "^1.0.12",
"jsdom": "^26.0.0",
"json-schema-to-zod": "^2.3.0",
"keyword-extractor": "^0.0.28",
"koffi": "^2.9.0",
"langfuse": "^3.38.4",
"langfuse-vercel": "^3.38.4",
"languagedetect": "^2.0.0",
"lodash": "^4.17.21",
"logsnag": "^1.0.0",
"luxon": "^3.4.3",
"mammoth": "^1.7.2",
"marked": "^14.1.2",
"md5": "^2.3.0",
"moment": "^2.29.4",
"ollama-ai-provider": "^1.2.0",
"openai": "^5.12.0",
"parse-diff": "^0.11.1",
"pdf-parse": "^1.1.1",
"pos": "^0.4.2",
"posthog-node": "^4.0.1",
"prettier": "^3.6.2",
"psl": "^1.15.0",
"rate-limiter-flexible": "2.4.2",
"redlock": "5.0.0-beta.2",
"resend": "^3.4.0",
"robots-parser": "^3.0.1",
"stripe": "^16.1.0",
"systeminformation": "^5.22.11",
"tldts": "^6.1.75",
"tough-cookie": "^4.1.4",
"turndown": "^7.1.3",
"turndown-plugin-gfm": "^1.0.2",
"undici": "^7.10.0",
"unstructured-client": "^0.11.3",
"uuid": "^10.0.0",
"winston": "^3.14.2",
"winston-transport": "^4.8.0",
"wordpos": "^2.1.0",
"ws": "^8.18.0",
"x402-express": "^0.5.3",
"xml2js": "^0.6.2",
"zod": "^3.24.2"
},
"nodemonConfig": {
"ignore": [
"*.docx",
"*.json",
"temp"
]
},
"pnpm": {
"onlyBuiltDependencies": [
"@sentry-internal/node-cpu-profiler",
"supabase"
],
"overrides": {
"brace-expansion@>=2.0.0 <=2.0.1": "2.0.2"
}
},
"lint-staged": {
"*.{js,jsx,ts,tsx,json,css,md}": "prettier --write"
}
}