History

Add change tracking support to Python and JS SDKs (#1448)

* Add change tracking support to Python and JS SDKs Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Replace test API keys with TEST_API_KEY placeholder Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Replace API keys with dummy values for testing Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Use environment variables for API keys in tests Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Move JS SDK test to correct location and add dependencies Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Remove old test file location Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Update test file to use TEST_API_KEY environment variable Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Update Python SDK test to use TEST_API_KEY environment variable Co-Authored-By: Nicolas Camara <nick@sideguide.dev> * Update package.json * Update __init__.py ---------

devin-ai-integration[bot] committed 1 year ago

Commit ef341391 parent 138a9757 files · +254 −20Browse files

Showing the changes to apps/python-sdk (3 of 7 files). Show all

Unified · Split

3 files
  1. apps/python-sdk/firecrawl/__init__.py +1 −1
  2. apps/python-sdk/firecrawl/firecrawl.py +15 −1
  3. apps/python-sdk/tests/test_change_tracking.py +98 −0
apps/python-sdk/firecrawl/__init__.py+1−1
@@ -13,7 +13,7 @@ import os · 12 unmodified lines
13 13
14from .firecrawl import FirecrawlApp # noqa14from .firecrawl import FirecrawlApp # noqa
15 15
16__version__ = "1.15.0"16__version__ = "1.16.0"
17 17
18# Define the logger for the Firecrawl project18# Define the logger for the Firecrawl project
19logger: logging.Logger = logging.getLogger("firecrawl")19logger: logging.Logger = logging.getLogger("firecrawl")
apps/python-sdk/firecrawl/firecrawl.py+15−1
@@ -75,6 +75,16 @@ class DeepResearchStatusResponse(pydantic.BaseModel): · 74 unmodified lines
75 sources: List[Dict[str, Any]]75 sources: List[Dict[str, Any]]
76 summaries: List[str]76 summaries: List[str]
77 77
78class ChangeTrackingData(pydantic.BaseModel):
79 """
80 Data for the change tracking format.
81 """
82 previousScrapeAt: Optional[str] = None
83 changeStatus: str # "new" | "same" | "changed" | "removed"
84 visibility: str # "visible" | "hidden"
85 diff: Optional[Dict[str, Any]] = None
86 json: Optional[Any] = None
87
78class FirecrawlApp:88class FirecrawlApp:
79 class SearchResponse(pydantic.BaseModel):89 class SearchResponse(pydantic.BaseModel):
80 """90 """
@@ -167,9 +177,13 @@ class FirecrawlApp: · 86 unmodified lines
167 json['schema'] = json['schema'].schema()177 json['schema'] = json['schema'].schema()
168 scrape_params['jsonOptions'] = json178 scrape_params['jsonOptions'] = json
169 179
180 change_tracking = params.get("changeTrackingOptions", {})
181 if change_tracking:
182 scrape_params['changeTrackingOptions'] = change_tracking
183
170 # Include any other params directly at the top level of scrape_params184 # Include any other params directly at the top level of scrape_params
171 for key, value in params.items():185 for key, value in params.items():
172 if key not in ['jsonOptions']:186 if key not in ['jsonOptions', 'changeTrackingOptions']:
173 scrape_params[key] = value187 scrape_params[key] = value
174 188
175 189
apps/python-sdk/tests/test_change_tracking.py+98−0
@@ -0,0 +1,98 @@
1import unittest
2from unittest.mock import patch, MagicMock
3import json
4import os
5from firecrawl import FirecrawlApp
6
7class TestChangeTracking(unittest.TestCase):
8 @patch('requests.post')
9 def test_change_tracking_format(self, mock_post):
10 mock_response = MagicMock()
11 mock_response.status_code = 200
12 mock_response.json.return_value = {
13 'success': True,
14 'data': {
15 'markdown': 'Test markdown content',
16 'changeTracking': {
17 'previousScrapeAt': '2023-01-01T00:00:00Z',
18 'changeStatus': 'changed',
19 'visibility': 'visible'
20 }
21 }
22 }
23 mock_post.return_value = mock_response
24
25 app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
26 result = app.scrape_url('https://example.com', {
27 'formats': ['markdown', 'changeTracking']
28 })
29
30 args, kwargs = mock_post.call_args
31 self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
32
33 self.assertEqual(result['changeTracking']['previousScrapeAt'], '2023-01-01T00:00:00Z')
34 self.assertEqual(result['changeTracking']['changeStatus'], 'changed')
35 self.assertEqual(result['changeTracking']['visibility'], 'visible')
36
37 @patch('requests.post')
38 def test_change_tracking_options(self, mock_post):
39 mock_response = MagicMock()
40 mock_response.status_code = 200
41 mock_response.json.return_value = {
42 'success': True,
43 'data': {
44 'markdown': 'Test markdown content',
45 'changeTracking': {
46 'previousScrapeAt': '2023-01-01T00:00:00Z',
47 'changeStatus': 'changed',
48 'visibility': 'visible',
49 'diff': {
50 'text': '@@ -1,1 +1,1 @@\n-old content\n+new content',
51 'json': {
52 'files': [{
53 'from': None,
54 'to': None,
55 'chunks': [{
56 'content': '@@ -1,1 +1,1 @@',
57 'changes': [{
58 'type': 'del',
59 'content': '-old content',
60 'del': True,
61 'ln': 1
62 }, {
63 'type': 'add',
64 'content': '+new content',
65 'add': True,
66 'ln': 1
67 }]
68 }]
69 }]
70 }
71 },
72 'json': {
73 'title': {
74 'previous': 'Old Title',
75 'current': 'New Title'
76 }
77 }
78 }
79 }
80 }
81 mock_post.return_value = mock_response
82
83 app = FirecrawlApp(api_key=os.environ.get('TEST_API_KEY', 'dummy-api-key-for-testing'))
84 result = app.scrape_url('https://example.com', {
85 'formats': ['markdown', 'changeTracking'],
86 'changeTrackingOptions': {
87 'modes': ['git-diff', 'json'],
88 'schema': {'type': 'object', 'properties': {'title': {'type': 'string'}}}
89 }
90 })
91
92 args, kwargs = mock_post.call_args
93 self.assertEqual(kwargs['json']['formats'], ['markdown', 'changeTracking'])
94 self.assertEqual(kwargs['json']['changeTrackingOptions']['modes'], ['git-diff', 'json'])
95
96 self.assertEqual(result['changeTracking']['diff']['text'], '@@ -1,1 +1,1 @@\n-old content\n+new content')
97 self.assertEqual(result['changeTracking']['json']['title']['previous'], 'Old Title')
98 self.assertEqual(result['changeTracking']['json']['title']['current'], 'New Title')