History

Finish LOVE scraper

Thibaut Courouble committed 10 years ago

Commit e3582e21 parent a25290d12 files · +117 −212Browse files

Showing the changes to lib (4 of 12 files). Show all

Unified · Split

4 files
  1. lib/docs/filters/core/internal_urls.rb +1 −0
  2. lib/docs/filters/love/clean_html.rb +37 −70
  3. lib/docs/filters/love/entries.rb +47 −13
  4. lib/docs/scrapers/love.rb +17 −73
lib/docs/filters/core/internal_urls.rb+1−0
@@ -39,6 +39,7 @@ module Docs · 38 unmodified lines
39 def to_internal_url(str)39 def to_internal_url(str)
40 return unless (url = parse_url(str)) && (subpath = subpath_to(url))40 return unless (url = parse_url(str)) && (subpath = subpath_to(url))
41 normalize_subpath(subpath)41 normalize_subpath(subpath)
42 subpath = URI.unescape(subpath) if context[:decode_and_clean_paths]
42 return if skip_subpath?(subpath)43 return if skip_subpath?(subpath)
43 normalize_url(url, subpath)44 normalize_url(url, subpath)
44 url45 url
lib/docs/filters/love/clean_html.rb+37−70
@@ -2,100 +2,67 @@ module Docs · 1 unmodified line
2 class Love2 class Love
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 # Fix syntax highlighting5 @doc = at_css('#mw-content-text')
6
6 css('.mw-code').each do |node|7 css('.mw-code').each do |node|
7 node.content = node.at_css("div > pre").content8 node.content = node.at_css('div > pre').content
8 node['data-language'] = 'lua'9 node['data-language'] = 'lua'
9 node.name = 'pre'10 node.name = 'pre'
10 end11 end
11 12
12 # Move header tags up13 css('span[id]').each do |node|
13 css('h2', 'h3').each do |node|14 node.parent['id'] = node['id']
14 headline = node.at_css('.mw-headline')15 node.before(node.children).remove
15 node['id'] = headline['id']
16 node.content = headline.inner_text
17 end16 end
18 17
19 # Move dt tags up18 css('table.notice').each do |node|
20 css('dt > span').each do |node|19 content = node.at_css('td:nth-child(2)').inner_html
21 node.parent.content = node.inner_text20 node.replace %(<p class="note">#{content}</p>)
22 end21 end
23 22
24 # Style notices and new/removed sections23 css('table.new-section', 'table.removed-section', 'table.removed-new-section').each do |node|
25 css('.notice', '.new-section', '.removed-section', '.removed-new-section').each do |node|24 klass = node['class'] == 'new-section' ? 'note-green' : 'note-red'
26 case node['class']25 content = node.css('td').map(&:inner_html).join('<br>')
27 when 'notice'26 node.replace %(<p class="note #{klass}">#{content}</p>)
28 node['class'] = 'note note-warning'
29 node.inner_html = node.at_css('td:nth-child(2)').inner_html
30 node.next.remove unless node.next.nil? or node.next.name != 'br'
31 when 'new-section', 'removed-section', 'removed-new-section'
32 node['class'] = node['class'] == 'new-section' ? 'note note-green' : 'note note-red'
33 node.inner_html = node.at_css('tr > td > i').inner_html \
34 + '<br>' \
35 + node.at_css('tr > td > small').inner_html
36 end
37
38 node.name = 'p'
39 node.remove_attribute('bgcolor')
40 node.remove_attribute('style')
41 node.remove_attribute('align')
42 end27 end
43 28
44 # Style new/removed features
45 css('.new-feature', '.removed-feature', '.removed-new-feature').each do |node|29 css('.new-feature', '.removed-feature', '.removed-new-feature').each do |node|
46 node.name = 'div'30 klass = node['class'] == 'new-feature' ? 'label-green' : 'label-red'
47 node['class'] = node['class'] == 'new-feature' ? 'box-heading label-green' : 'box-heading label-red'31 content = node.content.sub(' LÖVE', '')
48 node.remove_attribute('style')32 label = %( <span class="label #{klass}">#{content}</span>)
49 33
50 container = node.next_element34 node.next_element.css('dt').each { |n| n << label }
51 container.name = 'div'35 node.remove
52 container['class'] = 'box-with-heading'
53 container.remove_attribute('style')
54 end36 end
55 37
56 # Style tables38 css('img[src$="Add.png"]').each do |node|
57 css('table.smwtable').each do |table|39 node.parent['class'] = 'cell-green'
58 table.remove_attribute('style')40 node.remove
59 table.css('td').each do |cell|
60 cell.remove_attribute('style')
61 end
62 table.css('td:last-child', 'td:nth-last-child(2)').each do |cell|
63 img = cell.at_css('img')
64 if img then
65 if img['alt'] == 'Added since' then
66 cell['class'] = 'cell-green'
67 elsif img['alt'] == 'Removed in'
68 cell['class'] = 'cell-red'
69 end
70 img.remove
71 end
72 end
73 end41 end
74 42
75 # Remove Other Languages43 css('img[src$="Remove.png"]').each do |node|
76 css('#Other_Languages').remove44 node.parent['class'] = 'cell-red'
77 css('.i18n').remove45 node.remove
46 end
78 47
79 # Remove changelog48 css('table, tr, td, th').each do |node|
80 node = at_css('h2#Changelog')49 %w(style cellpadding cellspacing width height valign).each do |attribute|
81 if !node.nil? then50 node.remove_attribute(attribute)
82 begin51 end
83 nxt = node.next
84 node.remove
85 node = nxt
86 end while !node.nil? and node.name != 'h2'
87 end52 end
88 53
89 # Remove empty paragraphs54 css('.note i', '.note small', 'div:not([class])', '.smwtable td:nth-last-child(2) > a', '.smwtable td:last-child > a').each do |node|
90 css('p').each do |node|55 node.before(node.children).remove
91 node.remove if node.inner_text.strip == ''
92 end56 end
93 57
94 # Remove linebreaks that are the first or last child of a paragraph58 css('p > br').each do |node|
95 css('p > br:first-child', 'p > br:last-child').each do |node|59 node.parent.remove if node.parent.content.empty?
96 node.remove
97 end60 end
98 61
62 css('div > br', '> br', 'hr').remove
63 css('#Editing_the_wiki + p', '#Editing_the_wiki').remove
64 css('#Other_Languages', '.i18n').remove
65
99 doc66 doc
100 end67 end
101 end68 end
lib/docs/filters/love/entries.rb+47−13
@@ -1,21 +1,55 @@
1module Docs1module Docs
2 class Love2 class Love
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 TYPES = {
5 'require' => 'Lua',
6 'light_userdata' => 'Lua',
7 'value' => 'Lua',
8 'variable' => 'Lua',
9
10 'Audio_Formats' => 'love.sound',
11 'ImageFontFormat' => 'love.font',
12 'BlendMode_Formulas' => 'love.graphics',
13 'Shader_Variables' => 'love.graphics',
14
15 'AreaSpreadDistribution' => 'love.graphics',
16 'BodyType' => 'love.physics',
17 'BufferMode' => 'love.filesystem',
18 'CompressedFormat' => 'love.image',
19 'JoystickConstant' => 'love.joystick',
20 'ParticleInsertMode' => 'love.graphics',
21 'String' => 'love',
22 'TextureMode' => 'love.graphics'
23 }
24
25 def call
26 if context[:initial_paths].include?(slug)
27 css('table.smwtable td:first-child > a').each do |node|
28 TYPES[node.content.strip] = slug
29 end
30 end
31
32 super
33 end
34
4 def get_type35 def get_type
5 if TYPE_OVERRIDE.key?(slug) then36 if slug == 'love'
6 return TYPE_OVERRIDE[slug]37 'love'
7 elsif m = slug.match(/\A(love\.\w+)\z/) then38 elsif slug.start_with?('enet')
8 # modules and funcions39 'enet'
9 return LOVE_MODULES.include?(m[1]) ? m[1] : 'love'40 elsif slug.include?('Joint') || slug.include?('Shape')
10 elsif m = slug.match(/\A(love\.\w+)\.(\w+)/) then41 'love.physics'
11 # functions in modules42 elsif TYPES.key?(slug)
12 return m[1]43 TYPES[slug]
13 elsif context[:list_classes] and (m = slug.match(/\A\(?([A-Z]\w+)\)?(\:\w+)?/)) then44 elsif match = slug.match(/\A(love\.\w+)(\.\w+)?\z/)
14 # classes, their members and enums45 match[2] || context[:initial_paths].include?(match[1]) ? match[1] : 'love'
15 return m[1] unless m[1].include?('_')46 elsif at_css('#catlinks a[title="Category:Lua"]')
47 'Lua'
48 elsif
49 type = slug.split(':').first
50 type.remove! %r{[\(\)]}
51 TYPES[type]
16 end52 end
17 # usually this shouldn't happen
18 "Other"
19 end53 end
20 end54 end
21 end55 end
lib/docs/scrapers/love.rb+17−73
@@ -1,98 +1,42 @@
1module Docs1module Docs
2 class Love < UrlScraper2 class Love < UrlScraper
3 LOVE_MODULES = %w(
4 love
5 love.audio
6 love.event
7 love.filesystem
8 love.font
9 love.graphics
10 love.image
11 love.joystick
12 love.keyboard
13 love.math
14 love.mouse
15 love.physics
16 love.sound
17 love.system
18 love.thread
19 love.timer
20 love.touch
21 love.video
22 love.window
23 )
24 TYPE_OVERRIDE = {
25 "Audio_Formats" => "love.sound",
26 "ImageFontFormat" => "love.font",
27 "BlendMode_Formulas" => "BlendMode",
28 "Shader_Variables" => "Shader"
29 }
30
31 self.name = 'LÖVE'3 self.name = 'LÖVE'
32 self.slug = 'love'4 self.slug = 'love'
33 self.type = 'love'5 self.type = 'love'
6 self.release = '0.10.1'
34 self.base_url = 'https://love2d.org/wiki/'7 self.base_url = 'https://love2d.org/wiki/'
35 self.root_path = 'love'8 self.root_path = 'Main_Page'
36 self.initial_paths = LOVE_MODULES9 self.initial_paths = %w(love love.audio love.event love.filesystem love.font love.graphics
10 love.image love.joystick love.keyboard love.math love.mouse love.physics love.sound
11 love.system love.thread love.timer love.touch love.video love.window enet socket utf8)
37 self.links = {12 self.links = {
38 home: 'https://love2d.org/',13 home: 'https://love2d.org/',
39 code: 'https://bitbucket.org/rude/love'14 code: 'https://bitbucket.org/rude/love'
40 }15 }
41 16
42 html_filters.push 'love/clean_html', 'love/entries', 'title'17 html_filters.push 'love/entries', 'love/clean_html', 'title'
43
44 options[:root_title] = 'love'
45 options[:initial_paths] = LOVE_MODULES
46 18
19 options[:root_title] = 'LÖVE'
47 options[:decode_and_clean_paths] = true20 options[:decode_and_clean_paths] = true
21 options[:container] = '#bodyContent'
48 22
49 # Add types to classes and their members23 options[:skip] = %w(Getting_Started Building_LÖVE Tutorial Tutorials Game_Distribution License
50 options[:list_classes] = true24 Games Libraries Software Snippets Version_History Lovers PO2_Syndrome HSL_color Guidelines)
51 25
52 options[:container] = '#mw-content-text'
53
54 options[:only_patterns] = [
55 /\A(love\z|love\.|[A-Z]|\([A-Z])/
56 # love
57 # love.* (modules and functions)
58 # Uppercased (classes and enums)
59 # (Uppercased) (generalized classes)
60 ]
61 options[:skip] = %w(
62 Getting_Started
63 Building_LÖVE
64 Tutorial
65 Tutorials
66 Game_Distribution
67 License
68 Games
69 Libraries
70 Software
71 Snippets
72 Version_History
73 Lovers
74 PO2_Syndrome
75 HSL_color
76 )
77 options[:skip_patterns] = [26 options[:skip_patterns] = [
78 /_\([^\)]+\)\z/,27 /_\([^\)]+\)\z/, # anything_(language)
79 # anything_(language) (this might have to be tweaked)28 /\A(Special|Category|File|Help|Template|User|Tutorial):/,
80 /\ASpecial:/,29 /\A\d/
81 /\ACategory:/,
82 /\AFile:/,
83 /\AHelp:/,
84 /\ATemplate:/,
85 /\AUser:/,
86 /\ATutorial:/
87 # special pages are indistinguishable from instance methods
88 ]30 ]
89 31
90 options[:replace_paths] = {32 options[:replace_paths] = {
91 "Config_Files" => "love.conf"33 'Config_Files' => 'love.conf',
34 'conf.lua' => 'love.conf',
35 'lua-enet' => 'enet'
92 }36 }
93 37
94 options[:attribution] = <<-HTML38 options[:attribution] = <<-HTML
95 &copy; L&Ouml;VE Development Team<br>39 &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>
96 Licensed under the GNU Free Documentation License, Version 1.3.40 Licensed under the GNU Free Documentation License, Version 1.3.
97 HTML41 HTML
98 end42 end