History

Update & improve MDN scrapers

Closes 260.

Thibaut committed 11 years ago

Commit ded6daa1 parent aad00635 files · +58 −10Browse files

Unified · Split

5 files
  1. lib/docs/filters/css/entries.rb +12 −9
  2. lib/docs/filters/dom/entries.rb +35 −0
  3. lib/docs/filters/dom_events/entries.rb +9 −0
  4. lib/docs/scrapers/mdn/dom.rb +1 −1
  5. lib/docs/scrapers/mdn/dom_events.rb +1 −0
lib/docs/filters/css/entries.rb+12−9
@@ -9,9 +9,6 @@ module Docs · 8 unmodified lines
9 linear-gradient radial-gradient repeating-linear-gradient9 linear-gradient radial-gradient repeating-linear-gradient
10 repeating-radial-gradient var)10 repeating-radial-gradient var)
11 11
12 PSEUDO_ELEMENT_SLUGS = %w(::after ::before ::first-letter ::first-line
13 ::selection)
14
15 VALUE_SLUGS = %w(auto inherit initial none normal unset)12 VALUE_SLUGS = %w(auto inherit initial none normal unset)
16 13
17 ADDITIONAL_ENTRIES = {14 ADDITIONAL_ENTRIES = {
@@ -39,6 +36,7 @@ module Docs · 21 unmodified lines
39 'transform-function' => [36 'transform-function' => [
40 %w(matrix() matrix() Functions),37 %w(matrix() matrix() Functions),
41 %w(matrix3d() matrix3d() Functions),38 %w(matrix3d() matrix3d() Functions),
39 %w(perspective() perspective() Functions),
42 %w(rotate() rotate() Functions),40 %w(rotate() rotate() Functions),
43 %w(rotate3d() rotate3d() Functions),41 %w(rotate3d() rotate3d() Functions),
44 %w(rotateX() rotateX() Functions),42 %w(rotateX() rotateX() Functions),
@@ -60,18 +58,21 @@ module Docs · 15 unmodified lines
60 58
61 def get_name59 def get_name
62 case type60 case type
63 when 'Data Types' then "<#{super.remove ' value'}>"61 when 'Data Types' then "<#{super.remove ' value'}>"
64 when 'Functions' then "#{super}()"62 when 'Functions' then "#{super}()"
63 when 'Miscellaneous' then slug.to_s.gsub('_', ' ').gsub('/', ': ')
65 else super64 else super
66 end65 end
67 end66 end
68 67
69 def get_type68 def get_type
70 if slug.end_with? 'selectors'69 if slug.end_with?('selectors')
71 'Selectors'70 'Selectors'
72 elsif slug.start_with? ':'71 elsif slug.start_with?('::')
73 PSEUDO_ELEMENT_SLUGS.include?(slug) ? 'Pseudo-elements' : 'Pseudo-classes'72 'Pseudo-elements'
74 elsif slug.start_with? '@'73 elsif slug.start_with?(':')
74 'Pseudo-classes'
75 elsif slug.start_with?('@')
75 'At-rules'76 'At-rules'
76 elsif DATA_TYPE_SLUGS.include?(slug)77 elsif DATA_TYPE_SLUGS.include?(slug)
77 'Data Types'78 'Data Types'
@@ -79,6 +80,8 @@ module Docs · 1 unmodified line
79 'Functions'80 'Functions'
80 elsif VALUE_SLUGS.include?(slug)81 elsif VALUE_SLUGS.include?(slug)
81 'Values'82 'Values'
83 elsif slug =~ /\A[a-z]+_/i
84 'Miscellaneous'
82 else85 else
83 'Properties'86 'Properties'
84 end87 end
lib/docs/filters/dom/entries.rb+35−0
@@ -4,6 +4,7 @@ module Docs · 3 unmodified lines
4 TYPE_BY_SPEC = {4 TYPE_BY_SPEC = {
5 'Battery Status' => 'Battery Status',5 'Battery Status' => 'Battery Status',
6 'Canvas ' => 'Canvas',6 'Canvas ' => 'Canvas',
7 'CSS Font Loading' => 'CSS',
7 'CSS Object Model' => 'CSS',8 'CSS Object Model' => 'CSS',
8 'Cryptography' => 'Web Cryptography',9 'Cryptography' => 'Web Cryptography',
9 'Device Orientation' => 'Device Orientation',10 'Device Orientation' => 'Device Orientation',
@@ -19,6 +20,8 @@ module Docs · 9 unmodified lines
19 'Navigation Timing' => 'Navigation Timing',20 'Navigation Timing' => 'Navigation Timing',
20 'Network Information' => 'Network Information',21 'Network Information' => 'Network Information',
21 'Push API' => 'Push',22 'Push API' => 'Push',
23 'Shadow DOM' => 'Shadow DOM',
24 'Server-Sent Events' => 'Server-Sent Events',
22 'Service Workers' => 'Service Workers',25 'Service Workers' => 'Service Workers',
23 'Web Animations' => 'Animation',26 'Web Animations' => 'Animation',
24 'Web Audio' => 'Web Audio',27 'Web Audio' => 'Web Audio',
@@ -64,6 +67,8 @@ module Docs · 39 unmodified lines
64 'StyleSheet' => 'CSS',67 'StyleSheet' => 'CSS',
65 'Stylesheet' => 'CSS',68 'Stylesheet' => 'CSS',
66 'SVG' => 'SVG',69 'SVG' => 'SVG',
70 'timing' => 'Navigation Timing',
71 'Timing' => 'Navigation Timing',
67 'Touch' => 'Touch',72 'Touch' => 'Touch',
68 'TreeWalker' => 'TreeWalker',73 'TreeWalker' => 'TreeWalker',
69 'URL' => 'URL',74 'URL' => 'URL',
@@ -74,15 +79,22 @@ module Docs · 4 unmodified lines
74 TYPE_BY_NAME_INCLUDES = {79 TYPE_BY_NAME_INCLUDES = {
75 'ChildNode' => 'Node',80 'ChildNode' => 'Node',
76 'Crypto' => 'Web Cryptography',81 'Crypto' => 'Web Cryptography',
82 'FormData' => 'XMLHTTPRequest',
83 'ImageBitmap' => 'Canvas',
77 'ImageData' => 'Canvas',84 'ImageData' => 'Canvas',
78 'IndexedDB' => 'IndexedDB',85 'IndexedDB' => 'IndexedDB',
86 'Media Source' => 'Media',
79 'MediaStream' => 'Media',87 'MediaStream' => 'Media',
80 'NodeList' => 'Node',88 'NodeList' => 'Node',
81 'Path2D' => 'Canvas',89 'Path2D' => 'Canvas',
90 'Server-sent' => 'Server-Sent Events',
82 'ServiceWorker' => 'Service Workers',91 'ServiceWorker' => 'Service Workers',
83 'TextMetrics' => 'Canvas',92 'TextMetrics' => 'Canvas',
84 'udio' => 'Web Audio',93 'udio' => 'Web Audio',
94 'WebSocket' => 'Web Sockets',
85 'WebGL' => 'WebGL',95 'WebGL' => 'WebGL',
96 'WEBGL' => 'WebGL',
97 'WebRTC' => 'WebRTC',
86 'WebVR' => 'WebVR',98 'WebVR' => 'WebVR',
87 'Worker' => 'Web Workers' }99 'Worker' => 'Web Workers' }
88 100
@@ -111,6 +123,7 @@ module Docs · 22 unmodified lines
111 name.sub! 'API.', 'API: '123 name.sub! 'API.', 'API: '
112 # Comment.Comment => Comment.constructor124 # Comment.Comment => Comment.constructor
113 name.sub! %r{\A(\w+)\.\1\z}, '\1.constructor' unless name == 'window.window'125 name.sub! %r{\A(\w+)\.\1\z}, '\1.constructor' unless name == 'window.window'
126 name.prepend 'XMLHttpRequest.' if slug.start_with?('XMLHttpRequest/') && !name.start_with?('XMLHttpRequest')
114 name127 name
115 end128 end
116 129
@@ -160,6 +173,28 @@ module Docs · 43 unmodified lines
160 content = node.content173 content = node.content
161 SKIP_CONTENT.none? { |str| content.include?(str) }174 SKIP_CONTENT.none? { |str| content.include?(str) }
162 end175 end
176
177 def additional_entries
178 entries = []
179
180 if slug == 'history' || slug == 'XMLHttpRequest'
181 css('dt a[title*="not yet been written"]').each do |node|
182 next if node.parent.at_css('.obsolete')
183 name = node.content.sub('History', 'history')
184 id = node.parent['id'] = name.parameterize
185 entries << [name, id]
186 end
187 end
188
189 if slug == 'XMLHttpRequest'
190 css('h2[id="Methods_2"] ~ h3').each do |node|
191 break if node.content == 'Non-standard methods'
192 entries << ["#{name}.#{node.content}", node['id']]
193 end
194 end
195
196 entries
197 end
163 end198 end
164 end199 end
165end200end
lib/docs/filters/dom_events/entries.rb+9−0
@@ -3,12 +3,15 @@ module Docs · 2 unmodified lines
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 TYPE_BY_INFO = {4 TYPE_BY_INFO = {
5 'applicationCache' => 'Application Cache',5 'applicationCache' => 'Application Cache',
6 'Battery' => 'Battery',
6 'Clipboard' => 'Clipboard',7 'Clipboard' => 'Clipboard',
7 'CSS' => 'CSS',8 'CSS' => 'CSS',
8 'Drag' => 'Drag & Drop',9 'Drag' => 'Drag & Drop',
9 'Focus' => 'Focus',10 'Focus' => 'Focus',
10 'Fullscreen' => 'Fullscreen',11 'Fullscreen' => 'Fullscreen',
12 'Gamepad' => 'Gamepad',
11 'HashChange' => 'History',13 'HashChange' => 'History',
14 'IDB' => 'IndexedDB',
12 'IndexedDB' => 'IndexedDB',15 'IndexedDB' => 'IndexedDB',
13 'Keyboard' => 'Keyboard',16 'Keyboard' => 'Keyboard',
14 'edia' => 'Media',17 'edia' => 'Media',
@@ -16,6 +19,7 @@ module Docs · 1 unmodified line
16 'Offline' => 'Offline',19 'Offline' => 'Offline',
17 'Orientation' => 'Device',20 'Orientation' => 'Device',
18 'Sensor' => 'Device',21 'Sensor' => 'Device',
22 'SVG' => 'SVG',
19 'Page Visibility' => 'Page Visibility',23 'Page Visibility' => 'Page Visibility',
20 'Pointer' => 'Mouse',24 'Pointer' => 'Mouse',
21 'PopState' => 'History',25 'PopState' => 'History',
@@ -29,6 +33,7 @@ module Docs · 7 unmodified lines
29 'WebSocket' => 'WebSocket',33 'WebSocket' => 'WebSocket',
30 'Web Audio' => 'Web Audio',34 'Web Audio' => 'Web Audio',
31 'Web Messaging' => 'Web Messaging',35 'Web Messaging' => 'Web Messaging',
36 'WebRTC' => 'WebRTC',
32 'Wheel' => 'Mouse',37 'Wheel' => 'Mouse',
33 'Worker' => 'Web Workers' }38 'Worker' => 'Web Workers' }
34 39
@@ -61,6 +66,10 @@ module Docs · 26 unmodified lines
61 'Miscellaneous'66 'Miscellaneous'
62 end67 end
63 end68 end
69
70 def include_default_entry?
71 !doc.content.include?('Firefox OS specific')
72 end
64 end73 end
65 end74 end
66end75end
lib/docs/scrapers/mdn/dom.rb+1−1
@@ -54,7 +54,7 @@ module Docs · 53 unmodified lines
54 /\A\/InstallTrigger/,54 /\A\/InstallTrigger/,
55 /\A\/Entity/,55 /\A\/Entity/,
56 /\A\/Settings/,56 /\A\/Settings/,
57 /\A\/Telephony/,57 /telephony/i,
58 /\A\/NFC_API/,58 /\A\/NFC_API/,
59 /\A\/Window\/\w+bar/i,59 /\A\/Window\/\w+bar/i,
60 /\A\/Apps/,60 /\A\/Apps/,
lib/docs/scrapers/mdn/dom_events.rb+1−0
@@ -10,6 +10,7 @@ module Docs · 9 unmodified lines
10 options[:root_title] = 'DOM Events'10 options[:root_title] = 'DOM Events'
11 11
12 options[:skip] = %w(/MozOrientation)12 options[:skip] = %w(/MozOrientation)
13 options[:skip_patterns] = [/\A\/moz/i]
13 14
14 options[:fix_urls] = ->(url) do15 options[:fix_urls] = ->(url) do
15 url.sub! 'https://developer.mozilla.org/en-US/Mozilla_event_reference', DomEvents.base_url16 url.sub! 'https://developer.mozilla.org/en-US/Mozilla_event_reference', DomEvents.base_url