History

Improve DOM scraper

Thibaut committed 11 years ago

Commit 60f6c281 parent 018628e3 files · +53 −77Browse files

Unified · Split

3 files
  1. lib/docs/filters/dom/entries.rb +29 −7
  2. lib/docs/scrapers/mdn/dom.rb +23 −69
  3. lib/docs/scrapers/mdn/mdn.rb +1 −1
lib/docs/filters/dom/entries.rb+29−7
@@ -5,8 +5,11 @@ module Docs · 4 unmodified lines
5 'Battery Status' => 'Battery Status',5 'Battery Status' => 'Battery Status',
6 'Canvas ' => 'Canvas',6 'Canvas ' => 'Canvas',
7 'CSS Object Model' => 'CSS',7 'CSS Object Model' => 'CSS',
8 'Cryptography' => 'Web Cryptography',
8 'Device Orientation' => 'Device Orientation',9 'Device Orientation' => 'Device Orientation',
9 'Encoding' => 'Encoding',10 'Encoding' => 'Encoding',
11 'Encrypted Media Extensions' => 'Encrypted Media',
12 'Fetch' => 'Fetch',
10 'File API' => 'File',13 'File API' => 'File',
11 'Geolocation' => 'Geolocation',14 'Geolocation' => 'Geolocation',
12 'Media Capture' => 'Media',15 'Media Capture' => 'Media',
@@ -14,30 +17,35 @@ module Docs · 1 unmodified line
14 'MediaStream' => 'Media',17 'MediaStream' => 'Media',
15 'Navigation Timing' => 'Navigation Timing',18 'Navigation Timing' => 'Navigation Timing',
16 'Network Information' => 'Network Information',19 'Network Information' => 'Network Information',
20 'Push API' => 'Push',
17 'Service Workers' => 'Service Workers',21 'Service Workers' => 'Service Workers',
22 'Web Animations' => 'Animation',
18 'Web Audio' => 'Web Audio',23 'Web Audio' => 'Web Audio',
24 'Web Messaging' => 'Web Messaging',
19 'Web Storage' => 'Web Storage',25 'Web Storage' => 'Web Storage',
20 'Web Workers' => 'Web Workers',26 'Web Workers' => 'Web Workers',
21 'WebRTC' => 'WebRTC' }27 'WebRTC' => 'WebRTC' }
22 28
23 TYPE_BY_NAME_STARTS_WITH = {29 TYPE_BY_NAME_STARTS_WITH = {
24 'Audio' => 'Web Audio',30 'Audio' => 'Web Audio',
31 'Broadcast' => 'Broadcast Channel',
25 'Canvas' => 'Canvas',32 'Canvas' => 'Canvas',
26 'ChildNode' => 'Node',
27 'Console' => 'Console',
28 'CSS' => 'CSS',33 'CSS' => 'CSS',
34 'ChildNode' => 'Node',
35 'console' => 'Console',
29 'document' => 'Document',36 'document' => 'Document',
30 'DocumentFragment' => 'DocumentFragment',37 'DocumentFragment' => 'DocumentFragment',
31 'DOM' => 'DOM',38 'DOM' => 'DOM',
32 'element' => 'Element',39 'element' => 'Element',
33 'event' => 'Event',40 'event' => 'Event',
34 'Event' => 'Event',41 'Event' => 'Event',
42 'Fetch' => 'Fetch',
35 'File' => 'File',43 'File' => 'File',
36 'GlobalEventHandlers' => 'GlobalEventHandlers',44 'GlobalEventHandlers' => 'GlobalEventHandlers',
37 'history' => 'History',45 'history' => 'History',
38 'HTML' => 'Elements',46 'HTML' => 'Elements',
39 'IDB' => 'IndexedDB',47 'IDB' => 'IndexedDB',
40 'Location' => 'Location',48 'location' => 'Location',
41 'navigator' => 'Navigator',49 'navigator' => 'Navigator',
42 'MediaQuery' => 'MediaQuery',50 'MediaQuery' => 'MediaQuery',
43 'Node' => 'Node',51 'Node' => 'Node',
@@ -45,6 +53,7 @@ module Docs · 1 unmodified line
45 'ParentNode' => 'Node',53 'ParentNode' => 'Node',
46 'Range' => 'Range',54 'Range' => 'Range',
47 'RTC' => 'WebRTC',55 'RTC' => 'WebRTC',
56 'screen' => 'Screen',
48 'Selection' => 'Selection',57 'Selection' => 'Selection',
49 'Storage' => 'Web Storage',58 'Storage' => 'Web Storage',
50 'StyleSheet' => 'CSS',59 'StyleSheet' => 'CSS',
@@ -54,13 +63,17 @@ module Docs · 3 unmodified lines
54 'TreeWalker' => 'TreeWalker',63 'TreeWalker' => 'TreeWalker',
55 'Uint' => 'Typed Arrays',64 'Uint' => 'Typed Arrays',
56 'URL' => 'URL',65 'URL' => 'URL',
57 'window' => 'window',66 'window' => 'Window',
67 'Window' => 'Window',
58 'XMLHttpRequest' => 'XMLHTTPRequest' }68 'XMLHttpRequest' => 'XMLHTTPRequest' }
59 69
60 TYPE_BY_NAME_INCLUDES = {70 TYPE_BY_NAME_INCLUDES = {
71 'ChildNode' => 'Node',
72 'Crypto' => 'Web Cryptography',
61 'ImageData' => 'Canvas',73 'ImageData' => 'Canvas',
62 'IndexedDB' => 'IndexedDB',74 'IndexedDB' => 'IndexedDB',
63 'MediaStream' => 'Media',75 'MediaStream' => 'Media',
76 'NodeList' => 'Node',
64 'Path2D' => 'Canvas',77 'Path2D' => 'Canvas',
65 'ServiceWorker' => 'Service Workers',78 'ServiceWorker' => 'Service Workers',
66 'TextMetrics' => 'Canvas',79 'TextMetrics' => 'Canvas',
@@ -129,10 +142,19 @@ module Docs · 62 unmodified lines
129 end142 end
130 end143 end
131 144
145 SKIP_CONTENT = [
146 'not on a standards track',
147 'removed from the Web',
148 'not on a current W3C standards track',
149 'This feature is not built into all browsers',
150 'not currently supported in any browser'
151 ]
152
132 def include_default_entry?153 def include_default_entry?
133 (node = doc.at_css '.overheadIndicator').nil? ||154 return true if type == 'Console'
134 type == 'Console' ||155 return true unless node = doc.at_css('.overheadIndicator')
135 (node.content.exclude?('not on a standards track') && node.content.exclude?('removed from the Web'))156 content = node.content
157 SKIP_CONTENT.none? { |str| content.include?(str) }
136 end158 end
137 end159 end
138 end160 end
lib/docs/scrapers/mdn/dom.rb+23−69
@@ -2,53 +2,34 @@ module Docs · 1 unmodified line
2 class Dom < Mdn2 class Dom < Mdn
3 self.name = 'DOM'3 self.name = 'DOM'
4 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'4 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'
5 self.fix_redirections = true
5 6
6 html_filters.push 'dom/clean_html', 'dom/entries', 'title'7 html_filters.push 'dom/clean_html', 'dom/entries', 'title'
7 8
8 options[:root_title] = 'DOM'9 options[:root_title] = 'DOM'
9 10
10 # Don't want
11 options[:skip] = %w(11 options[:skip] = %w(
12 /App12 /Reference
13 /CallEvent13 /Index
14 /CanvasPixelArray14 /Document_Object_Model
15 /ChromeWorker15 /document/createProcessingInstruction
16 /document.createProcessingInstruction16 /document/documentURIObject
17 /document.documentURIObject17 /document/loadOverlay
18 /document.loadOverlay18 /document/tooltipNode
19 /document.tooltipNode
20 /DOMErrorHandler19 /DOMErrorHandler
21 /DOMLocator20 /DOMLocator
22 /DOMObject21 /DOMObject
23 /DOMStringList22 /DOMStringList
24 /Event/Comparison_of_Event_Targets23 /Event/Comparison_of_Event_Targets
25 /FMRadio
26 /IDBDatabaseException24 /IDBDatabaseException
27 /IndexedDB_API/Using_JavaScript_Generators_in_Firefox25 /IndexedDB_API/Using_JavaScript_Generators_in_Firefox
28 /NamedNodeMap
29 /Node.baseURIObject
30 /Node.nodePrincipal
31 /Notation26 /Notation
32 /PowerManager
33 /PushManager
34 /ProcessingInstruction27 /ProcessingInstruction
35 /TCPServerSocket
36 /TCPSocket
37 /TypeInfo28 /TypeInfo
38 /Using_the_Browser_API29 /window/getAttention
39 /Web_Video_Text_Tracks_Format30 /window/messageManager
40 /WifiManager31 /window/updateCommands
41 /window.controllers32 /window/pkcs11)
42 /window.crypto
43 /window.getAttention
44 /window.messageManager
45 /window.navigator.addIdleObserver
46 /window.navigator.getDeviceStorage
47 /window.navigator.getDeviceStorages
48 /window.navigator.removeIdleObserver
49 /window.navigator.requestWakeLock
50 /window.updateCommands
51 /window.pkcs11)
52 33
53 options[:skip_patterns] = [34 options[:skip_patterns] = [
54 /NS/,35 /NS/,
@@ -58,69 +39,42 @@ module Docs · 3 unmodified lines
58 /webkit/i,39 /webkit/i,
59 /gamepad/i,40 /gamepad/i,
60 /UserData/,41 /UserData/,
42 /Bluetooth/,
43 /FMRadio/i,
44 /XDomainRequest/i,
61 /\A\/Camera/,45 /\A\/Camera/,
62 /\A\/Data_Store_API/,46 /\A\/Data_Store_API/,
63 /\A\/DataStore/,47 /\A\/DataStore/,
64 /\A\/DeviceStorage/,48 /\A\/DeviceStorage/,
65 /\A\/DocumentTouch/,49 /\A\/DocumentTouch/,
66 /\A\/document\.xml/,50 /\A\/document\/xml/,
67 /\A\/XMLDocument/,51 /\A\/XMLDocument/,
68 /\A\/DOMCursor/,52 /\A\/DOMCursor/,
69 /\A\/DOMRequest/,53 /\A\/DOMRequest/,
70 /\A\/element\.on/,
71 /\A\/Entity/,54 /\A\/Entity/,
72 /\A\/navigator\.id/i,
73 /\A\/Settings/,55 /\A\/Settings/,
74 /\A\/Telephony/,56 /\A\/Telephony/,
75 /Bluetooth/,57 /\A\/Window\/\w+bar/i,
76 /\A\/Window\.\w+bar/i,
77 /\A\/Apps/,58 /\A\/Apps/,
78 /\A\/Contact/,59 /\A\/Contact/,
79 /\A\/L10n/,60 /\A\/L10n/,
80 /\A\/Permission/]61 /\A\/Permission/]
81 62
82 # Broken / Empty
83 options[:skip].concat %w(
84 /Attr.isId
85 /document.nodePrincipal
86 /Event/UIEvent
87 /Extensions
88 /StyleSheetList
89 /SVGPoint
90 /Window.dispatchEvent
91 /Window.restore
92 /Window.routeEvent
93 /Window.QueryInterface)
94
95 # Duplicates
96 options[:skip].concat %w(
97 /Reference
98 /Index
99 /form.elements
100 /select.type
101 /table.rows
102 /XMLHttpRequest/FormData
103 /Performance.now
104 /Document_Object_Model)
105
106 options[:fix_urls] = ->(url) do63 options[:fix_urls] = ->(url) do
107 return if url.include?('_') || url.include?('?')64 return if url.include?('_') || url.include?('?')
108 url.sub! 'https://developer.mozilla.org/en-US/docs/DOM/', "#{Dom.base_url}/"65 url.sub! 'https://developer.mozilla.org/en-US/docs/DOM/', "#{Dom.base_url}/"
109 url.sub! 'https://developer.mozilla.org/en/DOM/', "#{Dom.base_url}/"66 url.sub! 'https://developer.mozilla.org/en/DOM/', "#{Dom.base_url}/"
110 url.sub! "#{Dom.base_url}/Document\.", "#{Dom.base_url}/document."67 url.sub! "#{Dom.base_url}/Console", "#{Dom.base_url}/console"
68 url.sub! "#{Dom.base_url}/Document\/", "#{Dom.base_url}/document\/"
111 url.sub! "#{Dom.base_url}/Element", "#{Dom.base_url}/element"69 url.sub! "#{Dom.base_url}/Element", "#{Dom.base_url}/element"
112 url.sub! "#{Dom.base_url}/History", "#{Dom.base_url}/history"70 url.sub! "#{Dom.base_url}/History", "#{Dom.base_url}/history"
71 url.sub! "#{Dom.base_url}/Location", "#{Dom.base_url}/location"
113 url.sub! "#{Dom.base_url}/Navigator", "#{Dom.base_url}/navigator"72 url.sub! "#{Dom.base_url}/Navigator", "#{Dom.base_url}/navigator"
73 url.sub! "#{Dom.base_url}/Screen", "#{Dom.base_url}/screen"
74 url.sub! "#{Dom.base_url}/Window\/", "#{Dom.base_url}/window\/"
114 url.sub! "#{Dom.base_url}/notification", "#{Dom.base_url}/Notification"75 url.sub! "#{Dom.base_url}/notification", "#{Dom.base_url}/Notification"
115 url.sub! "#{Dom.base_url}/range", "#{Dom.base_url}/Range"76 url.sub! "#{Dom.base_url}/range", "#{Dom.base_url}/Range"
116 url.sub! "#{Dom.base_url}/Window", "#{Dom.base_url}/window"77 url.sub! "#{Dom.base_url}/event", "#{Dom.base_url}/Event"
117 url.sub! "#{Dom.base_url}/window.navigator", "#{Dom.base_url}/navigator"
118 url.sub! "#{Dom.base_url}/Selection/", "#{Dom.base_url}/Selection."
119 url.sub! "#{Dom.base_url}/windowTimers", "#{Dom.base_url}/window"
120 url.sub! "#{Dom.base_url}/windowEventHandlers", "#{Dom.base_url}/window"
121 url.sub! %r{\/windowLocalStorage(\.localStorage)?}i, "/window.localStorage"
122 url.sub! %r{\/windowSessionStorage(\.sessionStorage)?}i, "/window.sessionStorage"
123 url.sub! "#{Dom.base_url}/Screen.", "#{Dom.base_url}/window.screen"
124 url78 url
125 end79 end
126 end80 end
lib/docs/scrapers/mdn/mdn.rb+1−1
@@ -11,7 +11,7 @@ module Docs · 10 unmodified lines
11 11
12 options[:trailing_slash] = false12 options[:trailing_slash] = false
13 13
14 options[:skip_link] = ->(link) { link['title'].try(:include?, 'hasn\'t been written yet'.freeze) }14 options[:skip_link] = ->(link) { link['title'].try(:include?, 'written'.freeze) }
15 15
16 options[:attribution] = <<-HTML16 options[:attribution] = <<-HTML
17 &copy; 2015 Mozilla Contributors<br>17 &copy; 2015 Mozilla Contributors<br>