Update & improve MDN scrapers

Closes 260.

Thibaut committed 11 years ago

Commit ded6daa1 parent aad00635 files · +58 −10Browse files

Unified · Split

5 files

lib/docs/filters/css/entries.rb +12 −9
lib/docs/filters/dom/entries.rb +35 −0
lib/docs/filters/dom_events/entries.rb +9 −0
lib/docs/scrapers/mdn/dom.rb +1 −1
lib/docs/scrapers/mdn/dom_events.rb +1 −0

@@ -9,9 +9,6 @@ module Docs · 8 unmodified lines
9	9	linear-gradient radial-gradient repeating-linear-gradient
10	10	repeating-radial-gradient var)
11	11
12		− PSEUDO_ELEMENT_SLUGS = %w(::after ::before ::first-letter ::first-line
13		− ::selection)
14		−
15	12	VALUE_SLUGS = %w(auto inherit initial none normal unset)
16	13
17	14	ADDITIONAL_ENTRIES = {
@@ -39,6 +36,7 @@ module Docs · 21 unmodified lines
39	36	'transform-function' => [
40	37	%w(matrix() matrix() Functions),
41	38	%w(matrix3d() matrix3d() Functions),
	39	+ %w(perspective() perspective() Functions),
42	40	%w(rotate() rotate() Functions),
43	41	%w(rotate3d() rotate3d() Functions),
44	42	%w(rotateX() rotateX() Functions),
@@ -60,18 +58,21 @@ module Docs · 15 unmodified lines
60	58
61	59	def get_name
62	60	case type
63		− when 'Data Types' then "<#{super.remove ' value'}>"
64		− when 'Functions' then "#{super}()"
	61	+ when 'Data Types' then "<#{super.remove ' value'}>"
	62	+ when 'Functions' then "#{super}()"
	63	+ when 'Miscellaneous' then slug.to_s.gsub('_', ' ').gsub('/', ': ')
65	64	else super
66	65	end
67	66	end
68	67
69	68	def get_type
70		− if slug.end_with? 'selectors'
	69	+ if slug.end_with?('selectors')
71	70	'Selectors'
72		− elsif slug.start_with? ':'
73		− PSEUDO_ELEMENT_SLUGS.include?(slug) ? 'Pseudo-elements' : 'Pseudo-classes'
74		− elsif slug.start_with? '@'
	71	+ elsif slug.start_with?('::')
	72	+ 'Pseudo-elements'
	73	+ elsif slug.start_with?(':')
	74	+ 'Pseudo-classes'
	75	+ elsif slug.start_with?('@')
75	76	'At-rules'
76	77	elsif DATA_TYPE_SLUGS.include?(slug)
77	78	'Data Types'
@@ -79,6 +80,8 @@ module Docs · 1 unmodified line
79	80	'Functions'
80	81	elsif VALUE_SLUGS.include?(slug)
81	82	'Values'
	83	+ elsif slug =~ /\A[a-z]+_/i
	84	+ 'Miscellaneous'
82	85	else
83	86	'Properties'
84	87	end

lib/docs/filters/dom/entries.rb+35−0

@@ -4,6 +4,7 @@ module Docs · 3 unmodified lines
4	4	TYPE_BY_SPEC = {
5	5	'Battery Status' => 'Battery Status',
6	6	'Canvas ' => 'Canvas',
	7	+ 'CSS Font Loading' => 'CSS',
7	8	'CSS Object Model' => 'CSS',
8	9	'Cryptography' => 'Web Cryptography',
9	10	'Device Orientation' => 'Device Orientation',
@@ -19,6 +20,8 @@ module Docs · 9 unmodified lines
19	20	'Navigation Timing' => 'Navigation Timing',
20	21	'Network Information' => 'Network Information',
21	22	'Push API' => 'Push',
	23	+ 'Shadow DOM' => 'Shadow DOM',
	24	+ 'Server-Sent Events' => 'Server-Sent Events',
22	25	'Service Workers' => 'Service Workers',
23	26	'Web Animations' => 'Animation',
24	27	'Web Audio' => 'Web Audio',
@@ -64,6 +67,8 @@ module Docs · 39 unmodified lines
64	67	'StyleSheet' => 'CSS',
65	68	'Stylesheet' => 'CSS',
66	69	'SVG' => 'SVG',
	70	+ 'timing' => 'Navigation Timing',
	71	+ 'Timing' => 'Navigation Timing',
67	72	'Touch' => 'Touch',
68	73	'TreeWalker' => 'TreeWalker',
69	74	'URL' => 'URL',
@@ -74,15 +79,22 @@ module Docs · 4 unmodified lines
74	79	TYPE_BY_NAME_INCLUDES = {
75	80	'ChildNode' => 'Node',
76	81	'Crypto' => 'Web Cryptography',
	82	+ 'FormData' => 'XMLHTTPRequest',
	83	+ 'ImageBitmap' => 'Canvas',
77	84	'ImageData' => 'Canvas',
78	85	'IndexedDB' => 'IndexedDB',
	86	+ 'Media Source' => 'Media',
79	87	'MediaStream' => 'Media',
80	88	'NodeList' => 'Node',
81	89	'Path2D' => 'Canvas',
	90	+ 'Server-sent' => 'Server-Sent Events',
82	91	'ServiceWorker' => 'Service Workers',
83	92	'TextMetrics' => 'Canvas',
84	93	'udio' => 'Web Audio',
	94	+ 'WebSocket' => 'Web Sockets',
85	95	'WebGL' => 'WebGL',
	96	+ 'WEBGL' => 'WebGL',
	97	+ 'WebRTC' => 'WebRTC',
86	98	'WebVR' => 'WebVR',
87	99	'Worker' => 'Web Workers' }
88	100
@@ -111,6 +123,7 @@ module Docs · 22 unmodified lines
111	123	name.sub! 'API.', 'API: '
112	124	# Comment.Comment => Comment.constructor
113	125	name.sub! %r{\A(\w+)\.\1\z}, '\1.constructor' unless name == 'window.window'
	126	+ name.prepend 'XMLHttpRequest.' if slug.start_with?('XMLHttpRequest/') && !name.start_with?('XMLHttpRequest')
114	127	name
115	128	end
116	129
@@ -160,6 +173,28 @@ module Docs · 43 unmodified lines
160	173	content = node.content
161	174	SKIP_CONTENT.none? { \|str\| content.include?(str) }
162	175	end
	176	+
	177	+ def additional_entries
	178	+ entries = []
	179	+
	180	+ if slug == 'history' \|\| slug == 'XMLHttpRequest'
	181	+ css('dt a[title*="not yet been written"]').each do \|node\|
	182	+ next if node.parent.at_css('.obsolete')
	183	+ name = node.content.sub('History', 'history')
	184	+ id = node.parent['id'] = name.parameterize
	185	+ entries << [name, id]
	186	+ end
	187	+ end
	188	+
	189	+ if slug == 'XMLHttpRequest'
	190	+ css('h2[id="Methods_2"] ~ h3').each do \|node\|
	191	+ break if node.content == 'Non-standard methods'
	192	+ entries << ["#{name}.#{node.content}", node['id']]
	193	+ end
	194	+ end
	195	+
	196	+ entries
	197	+ end
163	198	end
164	199	end
165	200	end

lib/docs/filters/dom_events/entries.rb+9−0

@@ -3,12 +3,15 @@ module Docs · 2 unmodified lines
3	3	class EntriesFilter < Docs::EntriesFilter
4	4	TYPE_BY_INFO = {
5	5	'applicationCache' => 'Application Cache',
	6	+ 'Battery' => 'Battery',
6	7	'Clipboard' => 'Clipboard',
7	8	'CSS' => 'CSS',
8	9	'Drag' => 'Drag & Drop',
9	10	'Focus' => 'Focus',
10	11	'Fullscreen' => 'Fullscreen',
	12	+ 'Gamepad' => 'Gamepad',
11	13	'HashChange' => 'History',
	14	+ 'IDB' => 'IndexedDB',
12	15	'IndexedDB' => 'IndexedDB',
13	16	'Keyboard' => 'Keyboard',
14	17	'edia' => 'Media',
@@ -16,6 +19,7 @@ module Docs · 1 unmodified line
16	19	'Offline' => 'Offline',
17	20	'Orientation' => 'Device',
18	21	'Sensor' => 'Device',
	22	+ 'SVG' => 'SVG',
19	23	'Page Visibility' => 'Page Visibility',
20	24	'Pointer' => 'Mouse',
21	25	'PopState' => 'History',
@@ -29,6 +33,7 @@ module Docs · 7 unmodified lines
29	33	'WebSocket' => 'WebSocket',
30	34	'Web Audio' => 'Web Audio',
31	35	'Web Messaging' => 'Web Messaging',
	36	+ 'WebRTC' => 'WebRTC',
32	37	'Wheel' => 'Mouse',
33	38	'Worker' => 'Web Workers' }
34	39
@@ -61,6 +66,10 @@ module Docs · 26 unmodified lines
61	66	'Miscellaneous'
62	67	end
63	68	end
	69	+
	70	+ def include_default_entry?
	71	+ !doc.content.include?('Firefox OS specific')
	72	+ end
64	73	end
65	74	end
66	75	end

lib/docs/scrapers/mdn/dom.rb+1−1

@@ -54,7 +54,7 @@ module Docs · 53 unmodified lines
54	54	/\A\/InstallTrigger/,
55	55	/\A\/Entity/,
56	56	/\A\/Settings/,
57		− /\A\/Telephony/,
	57	+ /telephony/i,
58	58	/\A\/NFC_API/,
59	59	/\A\/Window\/\w+bar/i,
60	60	/\A\/Apps/,

lib/docs/scrapers/mdn/dom_events.rb+1−0

@@ -10,6 +10,7 @@ module Docs · 9 unmodified lines
10	10	options[:root_title] = 'DOM Events'
11	11
12	12	options[:skip] = %w(/MozOrientation)
	13	+ options[:skip_patterns] = [/\A\/moz/i]
13	14
14	15	options[:fix_urls] = ->(url) do
15	16	url.sub! 'https://developer.mozilla.org/en-US/Mozilla_event_reference', DomEvents.base_url