History

Improve DOM scraper

Thibaut committed 11 years ago

Commit 60f6c281 parent 018628e3 files · +53 −77Browse files

Unified · Split

3 files
  1. lib/docs/filters/dom/entries.rb +29 −7
  2. lib/docs/scrapers/mdn/dom.rb +23 −69
  3. lib/docs/scrapers/mdn/mdn.rb +1 −1
lib/docs/filters/dom/entries.rb+29−7
@@ -5,8 +5,11 @@ module Docs · 4 unmodified lines
55 'Battery Status' => 'Battery Status',
66 'Canvas ' => 'Canvas',
77 'CSS Object Model' => 'CSS',
8+ 'Cryptography' => 'Web Cryptography',
89 'Device Orientation' => 'Device Orientation',
910 'Encoding' => 'Encoding',
11+ 'Encrypted Media Extensions' => 'Encrypted Media',
12+ 'Fetch' => 'Fetch',
1013 'File API' => 'File',
1114 'Geolocation' => 'Geolocation',
1215 'Media Capture' => 'Media',
@@ -14,30 +17,35 @@ module Docs · 1 unmodified line
1417 'MediaStream' => 'Media',
1518 'Navigation Timing' => 'Navigation Timing',
1619 'Network Information' => 'Network Information',
20+ 'Push API' => 'Push',
1721 'Service Workers' => 'Service Workers',
22+ 'Web Animations' => 'Animation',
1823 'Web Audio' => 'Web Audio',
24+ 'Web Messaging' => 'Web Messaging',
1925 'Web Storage' => 'Web Storage',
2026 'Web Workers' => 'Web Workers',
2127 'WebRTC' => 'WebRTC' }
2228
2329 TYPE_BY_NAME_STARTS_WITH = {
2430 'Audio' => 'Web Audio',
31+ 'Broadcast' => 'Broadcast Channel',
2532 'Canvas' => 'Canvas',
26− 'ChildNode' => 'Node',
27− 'Console' => 'Console',
2833 'CSS' => 'CSS',
34+ 'ChildNode' => 'Node',
35+ 'console' => 'Console',
2936 'document' => 'Document',
3037 'DocumentFragment' => 'DocumentFragment',
3138 'DOM' => 'DOM',
3239 'element' => 'Element',
3340 'event' => 'Event',
3441 'Event' => 'Event',
42+ 'Fetch' => 'Fetch',
3543 'File' => 'File',
3644 'GlobalEventHandlers' => 'GlobalEventHandlers',
3745 'history' => 'History',
3846 'HTML' => 'Elements',
3947 'IDB' => 'IndexedDB',
40− 'Location' => 'Location',
48+ 'location' => 'Location',
4149 'navigator' => 'Navigator',
4250 'MediaQuery' => 'MediaQuery',
4351 'Node' => 'Node',
@@ -45,6 +53,7 @@ module Docs · 1 unmodified line
4553 'ParentNode' => 'Node',
4654 'Range' => 'Range',
4755 'RTC' => 'WebRTC',
56+ 'screen' => 'Screen',
4857 'Selection' => 'Selection',
4958 'Storage' => 'Web Storage',
5059 'StyleSheet' => 'CSS',
@@ -54,13 +63,17 @@ module Docs · 3 unmodified lines
5463 'TreeWalker' => 'TreeWalker',
5564 'Uint' => 'Typed Arrays',
5665 'URL' => 'URL',
57− 'window' => 'window',
66+ 'window' => 'Window',
67+ 'Window' => 'Window',
5868 'XMLHttpRequest' => 'XMLHTTPRequest' }
5969
6070 TYPE_BY_NAME_INCLUDES = {
71+ 'ChildNode' => 'Node',
72+ 'Crypto' => 'Web Cryptography',
6173 'ImageData' => 'Canvas',
6274 'IndexedDB' => 'IndexedDB',
6375 'MediaStream' => 'Media',
76+ 'NodeList' => 'Node',
6477 'Path2D' => 'Canvas',
6578 'ServiceWorker' => 'Service Workers',
6679 'TextMetrics' => 'Canvas',
@@ -129,10 +142,19 @@ module Docs · 62 unmodified lines
129142 end
130143 end
131144
145+ SKIP_CONTENT = [
146+ 'not on a standards track',
147+ 'removed from the Web',
148+ 'not on a current W3C standards track',
149+ 'This feature is not built into all browsers',
150+ 'not currently supported in any browser'
151+ ]
152+
132153 def include_default_entry?
133− (node = doc.at_css '.overheadIndicator').nil? ||
134− type == 'Console' ||
135− (node.content.exclude?('not on a standards track') && node.content.exclude?('removed from the Web'))
154+ return true if type == 'Console'
155+ return true unless node = doc.at_css('.overheadIndicator')
156+ content = node.content
157+ SKIP_CONTENT.none? { |str| content.include?(str) }
136158 end
137159 end
138160 end
lib/docs/scrapers/mdn/dom.rb+23−69
@@ -2,53 +2,34 @@ module Docs · 1 unmodified line
22 class Dom < Mdn
33 self.name = 'DOM'
44 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'
5+ self.fix_redirections = true
56
67 html_filters.push 'dom/clean_html', 'dom/entries', 'title'
78
89 options[:root_title] = 'DOM'
910
10− # Don't want
1111 options[:skip] = %w(
12− /App
13− /CallEvent
14− /CanvasPixelArray
15− /ChromeWorker
16− /document.createProcessingInstruction
17− /document.documentURIObject
18− /document.loadOverlay
19− /document.tooltipNode
12+ /Reference
13+ /Index
14+ /Document_Object_Model
15+ /document/createProcessingInstruction
16+ /document/documentURIObject
17+ /document/loadOverlay
18+ /document/tooltipNode
2019 /DOMErrorHandler
2120 /DOMLocator
2221 /DOMObject
2322 /DOMStringList
2423 /Event/Comparison_of_Event_Targets
25− /FMRadio
2624 /IDBDatabaseException
2725 /IndexedDB_API/Using_JavaScript_Generators_in_Firefox
28− /NamedNodeMap
29− /Node.baseURIObject
30− /Node.nodePrincipal
3126 /Notation
32− /PowerManager
33− /PushManager
3427 /ProcessingInstruction
35− /TCPServerSocket
36− /TCPSocket
3728 /TypeInfo
38− /Using_the_Browser_API
39− /Web_Video_Text_Tracks_Format
40− /WifiManager
41− /window.controllers
42− /window.crypto
43− /window.getAttention
44− /window.messageManager
45− /window.navigator.addIdleObserver
46− /window.navigator.getDeviceStorage
47− /window.navigator.getDeviceStorages
48− /window.navigator.removeIdleObserver
49− /window.navigator.requestWakeLock
50− /window.updateCommands
51− /window.pkcs11)
29+ /window/getAttention
30+ /window/messageManager
31+ /window/updateCommands
32+ /window/pkcs11)
5233
5334 options[:skip_patterns] = [
5435 /NS/,
@@ -58,69 +39,42 @@ module Docs · 3 unmodified lines
5839 /webkit/i,
5940 /gamepad/i,
6041 /UserData/,
42+ /Bluetooth/,
43+ /FMRadio/i,
44+ /XDomainRequest/i,
6145 /\A\/Camera/,
6246 /\A\/Data_Store_API/,
6347 /\A\/DataStore/,
6448 /\A\/DeviceStorage/,
6549 /\A\/DocumentTouch/,
66− /\A\/document\.xml/,
50+ /\A\/document\/xml/,
6751 /\A\/XMLDocument/,
6852 /\A\/DOMCursor/,
6953 /\A\/DOMRequest/,
70− /\A\/element\.on/,
7154 /\A\/Entity/,
72− /\A\/navigator\.id/i,
7355 /\A\/Settings/,
7456 /\A\/Telephony/,
75− /Bluetooth/,
76− /\A\/Window\.\w+bar/i,
57+ /\A\/Window\/\w+bar/i,
7758 /\A\/Apps/,
7859 /\A\/Contact/,
7960 /\A\/L10n/,
8061 /\A\/Permission/]
8162
82− # Broken / Empty
83− options[:skip].concat %w(
84− /Attr.isId
85− /document.nodePrincipal
86− /Event/UIEvent
87− /Extensions
88− /StyleSheetList
89− /SVGPoint
90− /Window.dispatchEvent
91− /Window.restore
92− /Window.routeEvent
93− /Window.QueryInterface)
94−
95− # Duplicates
96− options[:skip].concat %w(
97− /Reference
98− /Index
99− /form.elements
100− /select.type
101− /table.rows
102− /XMLHttpRequest/FormData
103− /Performance.now
104− /Document_Object_Model)
105−
10663 options[:fix_urls] = ->(url) do
10764 return if url.include?('_') || url.include?('?')
10865 url.sub! 'https://developer.mozilla.org/en-US/docs/DOM/', "#{Dom.base_url}/"
10966 url.sub! 'https://developer.mozilla.org/en/DOM/', "#{Dom.base_url}/"
110− url.sub! "#{Dom.base_url}/Document\.", "#{Dom.base_url}/document."
67+ url.sub! "#{Dom.base_url}/Console", "#{Dom.base_url}/console"
68+ url.sub! "#{Dom.base_url}/Document\/", "#{Dom.base_url}/document\/"
11169 url.sub! "#{Dom.base_url}/Element", "#{Dom.base_url}/element"
11270 url.sub! "#{Dom.base_url}/History", "#{Dom.base_url}/history"
71+ url.sub! "#{Dom.base_url}/Location", "#{Dom.base_url}/location"
11372 url.sub! "#{Dom.base_url}/Navigator", "#{Dom.base_url}/navigator"
73+ url.sub! "#{Dom.base_url}/Screen", "#{Dom.base_url}/screen"
74+ url.sub! "#{Dom.base_url}/Window\/", "#{Dom.base_url}/window\/"
11475 url.sub! "#{Dom.base_url}/notification", "#{Dom.base_url}/Notification"
11576 url.sub! "#{Dom.base_url}/range", "#{Dom.base_url}/Range"
116− url.sub! "#{Dom.base_url}/Window", "#{Dom.base_url}/window"
117− url.sub! "#{Dom.base_url}/window.navigator", "#{Dom.base_url}/navigator"
118− url.sub! "#{Dom.base_url}/Selection/", "#{Dom.base_url}/Selection."
119− url.sub! "#{Dom.base_url}/windowTimers", "#{Dom.base_url}/window"
120− url.sub! "#{Dom.base_url}/windowEventHandlers", "#{Dom.base_url}/window"
121− url.sub! %r{\/windowLocalStorage(\.localStorage)?}i, "/window.localStorage"
122− url.sub! %r{\/windowSessionStorage(\.sessionStorage)?}i, "/window.sessionStorage"
123− url.sub! "#{Dom.base_url}/Screen.", "#{Dom.base_url}/window.screen"
77+ url.sub! "#{Dom.base_url}/event", "#{Dom.base_url}/Event"
12478 url
12579 end
12680 end
lib/docs/scrapers/mdn/mdn.rb+1−1
@@ -11,7 +11,7 @@ module Docs · 10 unmodified lines
1111
1212 options[:trailing_slash] = false
1313
14− options[:skip_link] = ->(link) { link['title'].try(:include?, 'hasn\'t been written yet'.freeze) }
14+ options[:skip_link] = ->(link) { link['title'].try(:include?, 'written'.freeze) }
1515
1616 options[:attribution] = <<-HTML
1717 &copy; 2015 Mozilla Contributors<br>