Finish LOVE scraper

Thibaut Courouble committed 10 years ago

Commit e3582e21 parent a25290d12 files · +117 −212Browse files

Showing the changes to lib/docs (4 of 12 files). Show all

Unified · Split

4 files

lib/docs/filters/core/internal_urls.rb +1 −0
lib/docs/filters/love/clean_html.rb +37 −70
lib/docs/filters/love/entries.rb +47 −13
lib/docs/scrapers/love.rb +17 −73

lib/docs/filters/core/internal_urls.rb+1−0

@@ -39,6 +39,7 @@ module Docs · 38 unmodified lines
39	39	def to_internal_url(str)
40	40	return unless (url = parse_url(str)) && (subpath = subpath_to(url))
41	41	normalize_subpath(subpath)
	42	+ subpath = URI.unescape(subpath) if context[:decode_and_clean_paths]
42	43	return if skip_subpath?(subpath)
43	44	normalize_url(url, subpath)
44	45	url

lib/docs/filters/love/clean_html.rb+37−70

@@ -2,100 +2,67 @@ module Docs · 1 unmodified line
2	2	class Love
3	3	class CleanHtmlFilter < Filter
4	4	def call
5		− # Fix syntax highlighting
	5	+ @doc = at_css('#mw-content-text')
	6	+
6	7	css('.mw-code').each do \|node\|
7		− node.content = node.at_css("div > pre").content
	8	+ node.content = node.at_css('div > pre').content
8	9	node['data-language'] = 'lua'
9	10	node.name = 'pre'
10	11	end
11	12
12		− # Move header tags up
13		− css('h2', 'h3').each do \|node\|
14		− headline = node.at_css('.mw-headline')
15		− node['id'] = headline['id']
16		− node.content = headline.inner_text
	13	+ css('span[id]').each do \|node\|
	14	+ node.parent['id'] = node['id']
	15	+ node.before(node.children).remove
17	16	end
18	17
19		− # Move dt tags up
20		− css('dt > span').each do \|node\|
21		− node.parent.content = node.inner_text
	18	+ css('table.notice').each do \|node\|
	19	+ content = node.at_css('td:nth-child(2)').inner_html
	20	+ node.replace %(<p class="note">#{content}</p>)
22	21	end
23	22
24		− # Style notices and new/removed sections
25		− css('.notice', '.new-section', '.removed-section', '.removed-new-section').each do \|node\|
26		− case node['class']
27		− when 'notice'
28		− node['class'] = 'note note-warning'
29		− node.inner_html = node.at_css('td:nth-child(2)').inner_html
30		− node.next.remove unless node.next.nil? or node.next.name != 'br'
31		− when 'new-section', 'removed-section', 'removed-new-section'
32		− node['class'] = node['class'] == 'new-section' ? 'note note-green' : 'note note-red'
33		− node.inner_html = node.at_css('tr > td > i').inner_html \
34		− + '<br>' \
35		− + node.at_css('tr > td > small').inner_html
36		− end
37		−
38		− node.name = 'p'
39		− node.remove_attribute('bgcolor')
40		− node.remove_attribute('style')
41		− node.remove_attribute('align')
	23	+ css('table.new-section', 'table.removed-section', 'table.removed-new-section').each do \|node\|
	24	+ klass = node['class'] == 'new-section' ? 'note-green' : 'note-red'
	25	+ content = node.css('td').map(&:inner_html).join('<br>')
	26	+ node.replace %(<p class="note #{klass}">#{content}</p>)
42	27	end
43	28
44		− # Style new/removed features
45	29	css('.new-feature', '.removed-feature', '.removed-new-feature').each do \|node\|
46		− node.name = 'div'
47		− node['class'] = node['class'] == 'new-feature' ? 'box-heading label-green' : 'box-heading label-red'
48		− node.remove_attribute('style')
	30	+ klass = node['class'] == 'new-feature' ? 'label-green' : 'label-red'
	31	+ content = node.content.sub(' LÖVE', '')
	32	+ label = %( <span class="label #{klass}">#{content}</span>)
49	33
50		− container = node.next_element
51		− container.name = 'div'
52		− container['class'] = 'box-with-heading'
53		− container.remove_attribute('style')
	34	+ node.next_element.css('dt').each { \|n\| n << label }
	35	+ node.remove
54	36	end
55	37
56		− # Style tables
57		− css('table.smwtable').each do \|table\|
58		− table.remove_attribute('style')
59		− table.css('td').each do \|cell\|
60		− cell.remove_attribute('style')
61		− end
62		− table.css('td:last-child', 'td:nth-last-child(2)').each do \|cell\|
63		− img = cell.at_css('img')
64		− if img then
65		− if img['alt'] == 'Added since' then
66		− cell['class'] = 'cell-green'
67		− elsif img['alt'] == 'Removed in'
68		− cell['class'] = 'cell-red'
69		− end
70		− img.remove
71		− end
72		− end
	38	+ css('img[src$="Add.png"]').each do \|node\|
	39	+ node.parent['class'] = 'cell-green'
	40	+ node.remove
73	41	end
74	42
75		− # Remove Other Languages
76		− css('#Other_Languages').remove
77		− css('.i18n').remove
	43	+ css('img[src$="Remove.png"]').each do \|node\|
	44	+ node.parent['class'] = 'cell-red'
	45	+ node.remove
	46	+ end
78	47
79		− # Remove changelog
80		− node = at_css('h2#Changelog')
81		− if !node.nil? then
82		− begin
83		− nxt = node.next
84		− node.remove
85		− node = nxt
86		− end while !node.nil? and node.name != 'h2'
	48	+ css('table, tr, td, th').each do \|node\|
	49	+ %w(style cellpadding cellspacing width height valign).each do \|attribute\|
	50	+ node.remove_attribute(attribute)
	51	+ end
87	52	end
88	53
89		− # Remove empty paragraphs
90		− css('p').each do \|node\|
91		− node.remove if node.inner_text.strip == ''
	54	+ css('.note i', '.note small', 'div:not([class])', '.smwtable td:nth-last-child(2) > a', '.smwtable td:last-child > a').each do \|node\|
	55	+ node.before(node.children).remove
92	56	end
93	57
94		− # Remove linebreaks that are the first or last child of a paragraph
95		− css('p > br:first-child', 'p > br:last-child').each do \|node\|
96		− node.remove
	58	+ css('p > br').each do \|node\|
	59	+ node.parent.remove if node.parent.content.empty?
97	60	end
98	61
	62	+ css('div > br', '> br', 'hr').remove
	63	+ css('#Editing_the_wiki + p', '#Editing_the_wiki').remove
	64	+ css('#Other_Languages', '.i18n').remove
	65	+
99	66	doc
100	67	end
101	68	end

lib/docs/filters/love/entries.rb+47−13

@@ -1,21 +1,55 @@
1	1	module Docs
2	2	class Love
3	3	class EntriesFilter < Docs::EntriesFilter
	4	+ TYPES = {
	5	+ 'require' => 'Lua',
	6	+ 'light_userdata' => 'Lua',
	7	+ 'value' => 'Lua',
	8	+ 'variable' => 'Lua',
	9	+
	10	+ 'Audio_Formats' => 'love.sound',
	11	+ 'ImageFontFormat' => 'love.font',
	12	+ 'BlendMode_Formulas' => 'love.graphics',
	13	+ 'Shader_Variables' => 'love.graphics',
	14	+
	15	+ 'AreaSpreadDistribution' => 'love.graphics',
	16	+ 'BodyType' => 'love.physics',
	17	+ 'BufferMode' => 'love.filesystem',
	18	+ 'CompressedFormat' => 'love.image',
	19	+ 'JoystickConstant' => 'love.joystick',
	20	+ 'ParticleInsertMode' => 'love.graphics',
	21	+ 'String' => 'love',
	22	+ 'TextureMode' => 'love.graphics'
	23	+ }
	24	+
	25	+ def call
	26	+ if context[:initial_paths].include?(slug)
	27	+ css('table.smwtable td:first-child > a').each do \|node\|
	28	+ TYPES[node.content.strip] = slug
	29	+ end
	30	+ end
	31	+
	32	+ super
	33	+ end
	34	+
4	35	def get_type
5		− if TYPE_OVERRIDE.key?(slug) then
6		− return TYPE_OVERRIDE[slug]
7		− elsif m = slug.match(/\A(love\.\w+)\z/) then
8		− # modules and funcions
9		− return LOVE_MODULES.include?(m[1]) ? m[1] : 'love'
10		− elsif m = slug.match(/\A(love\.\w+)\.(\w+)/) then
11		− # functions in modules
12		− return m[1]
13		− elsif context[:list_classes] and (m = slug.match(/\A$?([A-Z]\w+)$?(\:\w+)?/)) then
14		− # classes, their members and enums
15		− return m[1] unless m[1].include?('_')
	36	+ if slug == 'love'
	37	+ 'love'
	38	+ elsif slug.start_with?('enet')
	39	+ 'enet'
	40	+ elsif slug.include?('Joint') \|\| slug.include?('Shape')
	41	+ 'love.physics'
	42	+ elsif TYPES.key?(slug)
	43	+ TYPES[slug]
	44	+ elsif match = slug.match(/\A(love\.\w+)(\.\w+)?\z/)
	45	+ match[2] \|\| context[:initial_paths].include?(match[1]) ? match[1] : 'love'
	46	+ elsif at_css('#catlinks a[title="Category:Lua"]')
	47	+ 'Lua'
	48	+ elsif
	49	+ type = slug.split(':').first
	50	+ type.remove! %r{[]}
	51	+ TYPES[type]
16	52	end
17		− # usually this shouldn't happen
18		− "Other"
19	53	end
20	54	end
21	55	end

lib/docs/scrapers/love.rb+17−73

@@ -1,98 +1,42 @@
1	1	module Docs
2	2	class Love < UrlScraper
3		− LOVE_MODULES = %w(
4		− love
5		− love.audio
6		− love.event
7		− love.filesystem
8		− love.font
9		− love.graphics
10		− love.image
11		− love.joystick
12		− love.keyboard
13		− love.math
14		− love.mouse
15		− love.physics
16		− love.sound
17		− love.system
18		− love.thread
19		− love.timer
20		− love.touch
21		− love.video
22		− love.window
23		− )
24		− TYPE_OVERRIDE = {
25		− "Audio_Formats" => "love.sound",
26		− "ImageFontFormat" => "love.font",
27		− "BlendMode_Formulas" => "BlendMode",
28		− "Shader_Variables" => "Shader"
29		− }
30		−
31	3	self.name = 'LÖVE'
32	4	self.slug = 'love'
33	5	self.type = 'love'
	6	+ self.release = '0.10.1'
34	7	self.base_url = 'https://love2d.org/wiki/'
35		− self.root_path = 'love'
36		− self.initial_paths = LOVE_MODULES
	8	+ self.root_path = 'Main_Page'
	9	+ self.initial_paths = %w(love love.audio love.event love.filesystem love.font love.graphics
	10	+ love.image love.joystick love.keyboard love.math love.mouse love.physics love.sound
	11	+ love.system love.thread love.timer love.touch love.video love.window enet socket utf8)
37	12	self.links = {
38	13	home: 'https://love2d.org/',
39	14	code: 'https://bitbucket.org/rude/love'
40	15	}
41	16
42		− html_filters.push 'love/clean_html', 'love/entries', 'title'
43		−
44		− options[:root_title] = 'love'
45		− options[:initial_paths] = LOVE_MODULES
	17	+ html_filters.push 'love/entries', 'love/clean_html', 'title'
46	18
	19	+ options[:root_title] = 'LÖVE'
47	20	options[:decode_and_clean_paths] = true
	21	+ options[:container] = '#bodyContent'
48	22
49		− # Add types to classes and their members
50		− options[:list_classes] = true
	23	+ options[:skip] = %w(Getting_Started Building_LÖVE Tutorial Tutorials Game_Distribution License
	24	+ Games Libraries Software Snippets Version_History Lovers PO2_Syndrome HSL_color Guidelines)
51	25
52		− options[:container] = '#mw-content-text'
53		−
54		− options[:only_patterns] = [
55		− /\A(love\z\|love\.\|[A-Z]\|\([A-Z])/
56		− # love
57		− # love.* (modules and functions)
58		− # Uppercased (classes and enums)
59		− # (Uppercased) (generalized classes)
60		− ]
61		− options[:skip] = %w(
62		− Getting_Started
63		− Building_LÖVE
64		− Tutorial
65		− Tutorials
66		− Game_Distribution
67		− License
68		− Games
69		− Libraries
70		− Software
71		− Snippets
72		− Version_History
73		− Lovers
74		− PO2_Syndrome
75		− HSL_color
76		− )
77	26	options[:skip_patterns] = [
78		− /_$[^$]+\)\z/,
79		− # anything_(language) (this might have to be tweaked)
80		− /\ASpecial:/,
81		− /\ACategory:/,
82		− /\AFile:/,
83		− /\AHelp:/,
84		− /\ATemplate:/,
85		− /\AUser:/,
86		− /\ATutorial:/
87		− # special pages are indistinguishable from instance methods
	27	+ /_$[^$]+\)\z/, # anything_(language)
	28	+ /\A(Special\|Category\|File\|Help\|Template\|User\|Tutorial):/,
	29	+ /\A\d/
88	30	]
89	31
90	32	options[:replace_paths] = {
91		− "Config_Files" => "love.conf"
	33	+ 'Config_Files' => 'love.conf',
	34	+ 'conf.lua' => 'love.conf',
	35	+ 'lua-enet' => 'enet'
92	36	}
93	37
94	38	options[:attribution] = <<-HTML
95		− © LÖVE Development Team<br>
	39	+ © 2006–2016 LÖVE Development Team<br>
96	40	Licensed under the GNU Free Documentation License, Version 1.3.
97	41	HTML
98	42	end