History

Add change tracking support to Python and JS SDKs (#1448)

* Add change tracking support to Python and JS SDKs Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Replace test API keys with TEST_API_KEY placeholder Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Replace API keys with dummy values for testing Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Use environment variables for API keys in tests Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Move JS SDK test to correct location and add dependencies Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Remove old test file location Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Update test file to use TEST_API_KEY environment variable Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Update Python SDK test to use TEST_API_KEY environment variable Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Update package.json * Update __init__.py ---------

devin-ai-integration[bot] committed 1 year ago

Commit ef341391 parent 138a9757 files · +254 −20Browse files

Showing the changes to apps/python-sdk (3 of 7 files). Show all

Unified · Split

3 files
  1. apps/python-sdk/firecrawl/__init__.py +1 −1
  2. apps/python-sdk/firecrawl/firecrawl.py +15 −1
  3. apps/python-sdk/tests/test_change_tracking.py +98 −0
apps/python-sdk/firecrawl/__init__.py+1−1
@@ -13,7 +13,7 @@ import os · 12 unmodified lines
1313
1414 from .firecrawl import FirecrawlApp # noqa
1515
16−__version__ = "1.15.0"
16+__version__ = "1.16.0"
1717
1818 # Define the logger for the Firecrawl project
1919 logger: logging.Logger = logging.getLogger("firecrawl")
apps/python-sdk/firecrawl/firecrawl.py+15−1
@@ -75,6 +75,16 @@ class DeepResearchStatusResponse(pydantic.BaseModel): · 74 unmodified lines
7575 sources: List[Dict[str, Any]]
7676 summaries: List[str]
7777
78+class ChangeTrackingData(pydantic.BaseModel):
79+ """
80+ Data for the change tracking format.
81+ """
82+ previousScrapeAt: Optional[str] = None
83+ changeStatus: str # "new" | "same" | "changed" | "removed"
84+ visibility: str # "visible" | "hidden"
85+ diff: Optional[Dict[str, Any]] = None
86+ json: Optional[Any] = None
87+
7888 class FirecrawlApp:
7989 class SearchResponse(pydantic.BaseModel):
8090 """
@@ -167,9 +177,13 @@ class FirecrawlApp: · 86 unmodified lines
167177 json['schema'] = json['schema'].schema()
168178 scrape_params['jsonOptions'] = json
169179
180+ change_tracking = params.get("changeTrackingOptions", {})
181+ if change_tracking:
182+ scrape_params['changeTrackingOptions'] = change_tracking
183+
170184 # Include any other params directly at the top level of scrape_params
171185 for key, value in params.items():
172− if key not in ['jsonOptions']:
186+ if key not in ['jsonOptions', 'changeTrackingOptions']:
173187 scrape_params[key] = value
174188
175189
apps/python-sdk/tests/test_change_tracking.py+98−0
@@ -0,0 +1,98 @@
1+import unittest
2+from unittest.mock import patch, MagicMock
3+import json
4+import os
5+from firecrawl import FirecrawlApp
6+
7+class TestChangeTracking(unittest.TestCase):
8+ @patch('requests.post')
9+ def test_change_tracking_format(self, mock_post):
10+ mock_response = MagicMock()
11+ mock_response.status_code = 200
12+ mock_response.json.return_value = {
13+ 'success': True,
14+ 'data': {
15+ 'markdown': 'Test markdown content',
16+ 'changeTracking': {
17+ 'previousScrapeAt': '2023-01-01T00:00:00Z',
18+ 'changeStatus': 'changed',
19+ 'visibility': 'visible'
20+ }
21+ }
22+ }
23+ mock_post.return_value = mock_response
24+
25+ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
26+ result = app.scrape_url('https://example.com', {
27+ 'formats': ['markdown', 'changeTracking']
28+ })
29+
30+ args, kwargs = mock_post.call_args
31+ self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
32+
33+ self.assertEqual(result['changeTracking']['previousScrapeAt'], '2023-01-01T00:00:00Z')
34+ self.assertEqual(result['changeTracking']['changeStatus'], 'changed')
35+ self.assertEqual(result['changeTracking']['visibility'], 'visible')
36+
37+ @patch('requests.post')
38+ def test_change_tracking_options(self, mock_post):
39+ mock_response = MagicMock()
40+ mock_response.status_code = 200
41+ mock_response.json.return_value = {
42+ 'success': True,
43+ 'data': {
44+ 'markdown': 'Test markdown content',
45+ 'changeTracking': {
46+ 'previousScrapeAt': '2023-01-01T00:00:00Z',
47+ 'changeStatus': 'changed',
48+ 'visibility': 'visible',
49+ 'diff': {
50+ 'text': '@@ -1,1 +1,1 @@\n-old content\n+new content',
51+ 'json': {
52+ 'files': [{
53+ 'from': None,
54+ 'to': None,
55+ 'chunks': [{
56+ 'content': '@@ -1,1 +1,1 @@',
57+ 'changes': [{
58+ 'type': 'del',
59+ 'content': '-old content',
60+ 'del': True,
61+ 'ln': 1
62+ }, {
63+ 'type': 'add',
64+ 'content': '+new content',
65+ 'add': True,
66+ 'ln': 1
67+ }]
68+ }]
69+ }]
70+ }
71+ },
72+ 'json': {
73+ 'title': {
74+ 'previous': 'Old Title',
75+ 'current': 'New Title'
76+ }
77+ }
78+ }
79+ }
80+ }
81+ mock_post.return_value = mock_response
82+
83+ app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
84+ result = app.scrape_url('https://example.com', {
85+ 'formats': ['markdown', 'changeTracking'],
86+ 'changeTrackingOptions': {
87+ 'modes': ['git-diff', 'json'],
88+ 'schema': {'type': 'object', 'properties': {'title': {'type': 'string'}}}
89+ }
90+ })
91+
92+ args, kwargs = mock_post.call_args
93+ self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
94+ self.assertEqual(kwargs['json']['changeTrackingOptions']['modes'], ['git-diff', 'json'])
95+
96+ self.assertEqual(result['changeTracking']['diff']['text'], '@@ -1,1 +1,1 @@\n-old content\n+new content')
97+ self.assertEqual(result['changeTracking']['json']['title']['previous'], 'Old Title')
98+ self.assertEqual(result['changeTracking']['json']['title']['current'], 'New Title')