History

Finish LOVE scraper

Thibaut Courouble committed 10 years ago

Commit e3582e21 parent a25290d12 files · +117 −212Browse files

Unified · Split

12 files
  1. assets/images/icons.png +0 −0
  2. assets/images/icons@2x.png +0 −0
  3. assets/javascripts/news.json +4 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. assets/stylesheets/pages/_love.scss +9 −59
  7. lib/docs/filters/core/internal_urls.rb +1 −0
  8. lib/docs/filters/love/clean_html.rb +37 −70
  9. lib/docs/filters/love/entries.rb +47 −13
  10. lib/docs/scrapers/love.rb +17 −73
  11. public/icons/docs/love/16.png +0 −0
  12. public/icons/docs/love/16@2x.png +0 −0
assets/images/icons.png+0−0

Binary file not shown.

assets/images/icons@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+4−1
@@ -1,10 +1,13 @@
1[1[
2 [2 [
3 "2016-06-19",
4 "New documentation: <a href=\"/love/\">L&Ouml;VE</a>"
5 ], [
3 "2016-06-12",6 "2016-06-12",
4 "New documentation: <a href=\"/angular/\">Angular 2</a>"7 "New documentation: <a href=\"/angular/\">Angular 2</a>"
5 ], [8 ], [
6 "2016-06-05",9 "2016-06-05",
7 "New documentation: <a href=\"/kotlin/\">Kotlin</a> and <a href=\"/padrino/\">Padrino</a>"10 "New documentations: <a href=\"/kotlin/\">Kotlin</a> and <a href=\"/padrino/\">Padrino</a>"
8 ], [11 ], [
9 "2016-04-24",12 "2016-04-24",
10 "New documentations: <a href=\"/numpy/\">NumPy</a> and <a href=\"/apache_pig/\">Apache Pig</a>"13 "New documentations: <a href=\"/numpy/\">NumPy</a> and <a href=\"/apache_pig/\">Apache Pig</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -275,6 +275,11 @@ credits = [ · 274 unmodified lines
275 '1994–2015 Lua.org, PUC-Rio',275 '1994–2015 Lua.org, PUC-Rio',
276 'MIT',276 'MIT',
277 'http://www.lua.org/license.html'277 'http://www.lua.org/license.html'
278 ], [
279 'L&Ouml;VE',
280 '2006-2016 L&Ouml;VE Development Team',
281 'GFDL',
282 'http://www.gnu.org/copyleft/fdl.html'
278 ], [283 ], [
279 'Marionette.js',284 'Marionette.js',
280 '2016 Muted Solutions, LLC',285 '2016 Muted Solutions, LLC',
assets/stylesheets/global/_icons.scss+1−0
@@ -142,3 +142,4 @@ · 141 unmodified lines
142._icon-kotlin:before { background-position: -6rem -11rem; }142._icon-kotlin:before { background-position: -6rem -11rem; }
143._icon-padrino:before { background-position: -7rem -11rem; }143._icon-padrino:before { background-position: -7rem -11rem; }
144._icon-angular:before { background-position: -8rem -11rem; }144._icon-angular:before { background-position: -8rem -11rem; }
145._icon-love:before { background-position: -9rem -11rem; }
assets/stylesheets/pages/_love.scss+9−59
@@ -1,67 +1,17 @@
1._love {1._love {
2 padding-left: 1rem;2 @extend %simple;
3 3
4 h1, h2 { margin-left: -1rem; }
5 h2 { @extend %block-heading; }
6 h3 { margin-left: -0.5rem; @extend %block-label; }
7
8 ._mobile & {
9 padding-left: 0;
10
11 h1, h2, h3 { margin-left: 0; }
12 }
13
14 p > code, li > code { @extend %label; }
15 blockquote { @extend %note; }
16
17 .box { @extend %box; }
18 .note { @extend %note; }4 .note { @extend %note; }
19 .label { @extend %label; }5 .note-green { @extend %note-green; }
6 .note-red { @extend %note-red; }
20 7
21 .box-heading {8 .label, dt > code { @extend %label; }
22 @extend %heading-box;9 .label-green { @extend %label-green; }
23 padding: .5em .75em;10 .label-red { @extend %label-red; }
24 margin-top: 1.5rem;11
25 margin-bottom: 0px;12 .smwtable { width: 100%; }
26 border-bottom: none;13 .smwtable td:nth-last-child(2), .smwtable td:last-child { width: 2.5em; }
27 border-bottom-left-radius: 0px;
28 border-bottom-right-radius: 0px;
29 }
30 .box-with-heading {
31 @extend %box;
32 padding: .5em .75em;
33 margin-top: 0px;
34 margin-bottom: 1.5rem;
35 border-top-left-radius: 0px;
36 border-top-right-radius: 0px;
37 }
38 14
39 .note-green, .label-green { @extend %note-green; }
40 .note-red, .label-red { @extend %note-red; }
41 .note-orange, .label-orange { @extend %note-orange; }
42 .cell-green { background: $noteGreenBackground; }15 .cell-green { background: $noteGreenBackground; }
43 .cell-red { background: $noteRedBackground; }16 .cell-red { background: $noteRedBackground; }
44
45 .smwtable {
46 width: 100%;
47 tr {
48 td {
49 word-wrap: break-word;
50 }
51
52 td:first-child, td:nth-last-child(2), td:last-child {
53 vertical-align: middle;
54 white-space: nowrap;
55 width: 1em;
56 overflow: hidden;
57 }
58 }
59 }
60
61 hr {
62 border: none;
63 height: 1px;
64 background-color: $textColorLighter;
65 margin: 1.5em 0 1em;
66 }
67}17}
lib/docs/filters/core/internal_urls.rb+1−0
@@ -39,6 +39,7 @@ module Docs · 38 unmodified lines
39 def to_internal_url(str)39 def to_internal_url(str)
40 return unless (url = parse_url(str)) && (subpath = subpath_to(url))40 return unless (url = parse_url(str)) && (subpath = subpath_to(url))
41 normalize_subpath(subpath)41 normalize_subpath(subpath)
42 subpath = URI.unescape(subpath) if context[:decode_and_clean_paths]
42 return if skip_subpath?(subpath)43 return if skip_subpath?(subpath)
43 normalize_url(url, subpath)44 normalize_url(url, subpath)
44 url45 url
lib/docs/filters/love/clean_html.rb+37−70
@@ -2,100 +2,67 @@ module Docs · 1 unmodified line
2 class Love2 class Love
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 # Fix syntax highlighting5 @doc = at_css('#mw-content-text')
6
6 css('.mw-code').each do |node|7 css('.mw-code').each do |node|
7 node.content = node.at_css("div > pre").content8 node.content = node.at_css('div > pre').content
8 node['data-language'] = 'lua'9 node['data-language'] = 'lua'
9 node.name = 'pre'10 node.name = 'pre'
10 end11 end
11 12
12 # Move header tags up13 css('span[id]').each do |node|
13 css('h2', 'h3').each do |node|14 node.parent['id'] = node['id']
14 headline = node.at_css('.mw-headline')15 node.before(node.children).remove
15 node['id'] = headline['id']
16 node.content = headline.inner_text
17 end16 end
18 17
19 # Move dt tags up18 css('table.notice').each do |node|
20 css('dt > span').each do |node|19 content = node.at_css('td:nth-child(2)').inner_html
21 node.parent.content = node.inner_text20 node.replace %(<p class="note">#{content}</p>)
22 end21 end
23 22
24 # Style notices and new/removed sections23 css('table.new-section', 'table.removed-section', 'table.removed-new-section').each do |node|
25 css('.notice', '.new-section', '.removed-section', '.removed-new-section').each do |node|24 klass = node['class'] == 'new-section' ? 'note-green' : 'note-red'
26 case node['class']25 content = node.css('td').map(&:inner_html).join('<br>')
27 when 'notice'26 node.replace %(<p class="note #{klass}">#{content}</p>)
28 node['class'] = 'note note-warning'
29 node.inner_html = node.at_css('td:nth-child(2)').inner_html
30 node.next.remove unless node.next.nil? or node.next.name != 'br'
31 when 'new-section', 'removed-section', 'removed-new-section'
32 node['class'] = node['class'] == 'new-section' ? 'note note-green' : 'note note-red'
33 node.inner_html = node.at_css('tr > td > i').inner_html \
34 + '<br>' \
35 + node.at_css('tr > td > small').inner_html
36 end
37
38 node.name = 'p'
39 node.remove_attribute('bgcolor')
40 node.remove_attribute('style')
41 node.remove_attribute('align')
42 end27 end
43 28
44 # Style new/removed features
45 css('.new-feature', '.removed-feature', '.removed-new-feature').each do |node|29 css('.new-feature', '.removed-feature', '.removed-new-feature').each do |node|
46 node.name = 'div'30 klass = node['class'] == 'new-feature' ? 'label-green' : 'label-red'
47 node['class'] = node['class'] == 'new-feature' ? 'box-heading label-green' : 'box-heading label-red'31 content = node.content.sub(' LÖVE', '')
48 node.remove_attribute('style')32 label = %( <span class="label #{klass}">#{content}</span>)
49 33
50 container = node.next_element34 node.next_element.css('dt').each { |n| n << label }
51 container.name = 'div'35 node.remove
52 container['class'] = 'box-with-heading'
53 container.remove_attribute('style')
54 end36 end
55 37
56 # Style tables38 css('img[src$="Add.png"]').each do |node|
57 css('table.smwtable').each do |table|39 node.parent['class'] = 'cell-green'
58 table.remove_attribute('style')40 node.remove
59 table.css('td').each do |cell|
60 cell.remove_attribute('style')
61 end
62 table.css('td:last-child', 'td:nth-last-child(2)').each do |cell|
63 img = cell.at_css('img')
64 if img then
65 if img['alt'] == 'Added since' then
66 cell['class'] = 'cell-green'
67 elsif img['alt'] == 'Removed in'
68 cell['class'] = 'cell-red'
69 end
70 img.remove
71 end
72 end
73 end41 end
74 42
75 # Remove Other Languages43 css('img[src$="Remove.png"]').each do |node|
76 css('#Other_Languages').remove44 node.parent['class'] = 'cell-red'
77 css('.i18n').remove45 node.remove
46 end
78 47
79 # Remove changelog48 css('table, tr, td, th').each do |node|
80 node = at_css('h2#Changelog')49 %w(style cellpadding cellspacing width height valign).each do |attribute|
81 if !node.nil? then50 node.remove_attribute(attribute)
82 begin51 end
83 nxt = node.next
84 node.remove
85 node = nxt
86 end while !node.nil? and node.name != 'h2'
87 end52 end
88 53
89 # Remove empty paragraphs54 css('.note i', '.note small', 'div:not([class])', '.smwtable td:nth-last-child(2) > a', '.smwtable td:last-child > a').each do |node|
90 css('p').each do |node|55 node.before(node.children).remove
91 node.remove if node.inner_text.strip == ''
92 end56 end
93 57
94 # Remove linebreaks that are the first or last child of a paragraph58 css('p > br').each do |node|
95 css('p > br:first-child', 'p > br:last-child').each do |node|59 node.parent.remove if node.parent.content.empty?
96 node.remove
97 end60 end
98 61
62 css('div > br', '> br', 'hr').remove
63 css('#Editing_the_wiki + p', '#Editing_the_wiki').remove
64 css('#Other_Languages', '.i18n').remove
65
99 doc66 doc
100 end67 end
101 end68 end
lib/docs/filters/love/entries.rb+47−13
@@ -1,21 +1,55 @@
1module Docs1module Docs
2 class Love2 class Love
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 TYPES = {
5 'require' => 'Lua',
6 'light_userdata' => 'Lua',
7 'value' => 'Lua',
8 'variable' => 'Lua',
9
10 'Audio_Formats' => 'love.sound',
11 'ImageFontFormat' => 'love.font',
12 'BlendMode_Formulas' => 'love.graphics',
13 'Shader_Variables' => 'love.graphics',
14
15 'AreaSpreadDistribution' => 'love.graphics',
16 'BodyType' => 'love.physics',
17 'BufferMode' => 'love.filesystem',
18 'CompressedFormat' => 'love.image',
19 'JoystickConstant' => 'love.joystick',
20 'ParticleInsertMode' => 'love.graphics',
21 'String' => 'love',
22 'TextureMode' => 'love.graphics'
23 }
24
25 def call
26 if context[:initial_paths].include?(slug)
27 css('table.smwtable td:first-child > a').each do |node|
28 TYPES[node.content.strip] = slug
29 end
30 end
31
32 super
33 end
34
4 def get_type35 def get_type
5 if TYPE_OVERRIDE.key?(slug) then36 if slug == 'love'
6 return TYPE_OVERRIDE[slug]37 'love'
7 elsif m = slug.match(/\A(love\.\w+)\z/) then38 elsif slug.start_with?('enet')
8 # modules and funcions39 'enet'
9 return LOVE_MODULES.include?(m[1]) ? m[1] : 'love'40 elsif slug.include?('Joint') || slug.include?('Shape')
10 elsif m = slug.match(/\A(love\.\w+)\.(\w+)/) then41 'love.physics'
11 # functions in modules42 elsif TYPES.key?(slug)
12 return m[1]43 TYPES[slug]
13 elsif context[:list_classes] and (m = slug.match(/\A\(?([A-Z]\w+)\)?(\:\w+)?/)) then44 elsif match = slug.match(/\A(love\.\w+)(\.\w+)?\z/)
14 # classes, their members and enums45 match[2] || context[:initial_paths].include?(match[1]) ? match[1] : 'love'
15 return m[1] unless m[1].include?('_')46 elsif at_css('#catlinks a[title="Category:Lua"]')
47 'Lua'
48 elsif
49 type = slug.split(':').first
50 type.remove! %r{[\(\)]}
51 TYPES[type]
16 end52 end
17 # usually this shouldn't happen
18 "Other"
19 end53 end
20 end54 end
21 end55 end
lib/docs/scrapers/love.rb+17−73
@@ -1,98 +1,42 @@
1module Docs1module Docs
2 class Love < UrlScraper2 class Love < UrlScraper
3 LOVE_MODULES = %w(
4 love
5 love.audio
6 love.event
7 love.filesystem
8 love.font
9 love.graphics
10 love.image
11 love.joystick
12 love.keyboard
13 love.math
14 love.mouse
15 love.physics
16 love.sound
17 love.system
18 love.thread
19 love.timer
20 love.touch
21 love.video
22 love.window
23 )
24 TYPE_OVERRIDE = {
25 "Audio_Formats" => "love.sound",
26 "ImageFontFormat" => "love.font",
27 "BlendMode_Formulas" => "BlendMode",
28 "Shader_Variables" => "Shader"
29 }
30
31 self.name = 'LÖVE'3 self.name = 'LÖVE'
32 self.slug = 'love'4 self.slug = 'love'
33 self.type = 'love'5 self.type = 'love'
6 self.release = '0.10.1'
34 self.base_url = 'https://love2d.org/wiki/'7 self.base_url = 'https://love2d.org/wiki/'
35 self.root_path = 'love'8 self.root_path = 'Main_Page'
36 self.initial_paths = LOVE_MODULES9 self.initial_paths = %w(love love.audio love.event love.filesystem love.font love.graphics
10 love.image love.joystick love.keyboard love.math love.mouse love.physics love.sound
11 love.system love.thread love.timer love.touch love.video love.window enet socket utf8)
37 self.links = {12 self.links = {
38 home: 'https://love2d.org/',13 home: 'https://love2d.org/',
39 code: 'https://bitbucket.org/rude/love'14 code: 'https://bitbucket.org/rude/love'
40 }15 }
41 16
42 html_filters.push 'love/clean_html', 'love/entries', 'title'17 html_filters.push 'love/entries', 'love/clean_html', 'title'
43
44 options[:root_title] = 'love'
45 options[:initial_paths] = LOVE_MODULES
46 18
19 options[:root_title] = 'LÖVE'
47 options[:decode_and_clean_paths] = true20 options[:decode_and_clean_paths] = true
21 options[:container] = '#bodyContent'
48 22
49 # Add types to classes and their members23 options[:skip] = %w(Getting_Started Building_LÖVE Tutorial Tutorials Game_Distribution License
50 options[:list_classes] = true24 Games Libraries Software Snippets Version_History Lovers PO2_Syndrome HSL_color Guidelines)
51 25
52 options[:container] = '#mw-content-text'
53
54 options[:only_patterns] = [
55 /\A(love\z|love\.|[A-Z]|\([A-Z])/
56 # love
57 # love.* (modules and functions)
58 # Uppercased (classes and enums)
59 # (Uppercased) (generalized classes)
60 ]
61 options[:skip] = %w(
62 Getting_Started
63 Building_LÖVE
64 Tutorial
65 Tutorials
66 Game_Distribution
67 License
68 Games
69 Libraries
70 Software
71 Snippets
72 Version_History
73 Lovers
74 PO2_Syndrome
75 HSL_color
76 )
77 options[:skip_patterns] = [26 options[:skip_patterns] = [
78 /_\([^\)]+\)\z/,27 /_\([^\)]+\)\z/, # anything_(language)
79 # anything_(language) (this might have to be tweaked)28 /\A(Special|Category|File|Help|Template|User|Tutorial):/,
80 /\ASpecial:/,29 /\A\d/
81 /\ACategory:/,
82 /\AFile:/,
83 /\AHelp:/,
84 /\ATemplate:/,
85 /\AUser:/,
86 /\ATutorial:/
87 # special pages are indistinguishable from instance methods
88 ]30 ]
89 31
90 options[:replace_paths] = {32 options[:replace_paths] = {
91 "Config_Files" => "love.conf"33 'Config_Files' => 'love.conf',
34 'conf.lua' => 'love.conf',
35 'lua-enet' => 'enet'
92 }36 }
93 37
94 options[:attribution] = <<-HTML38 options[:attribution] = <<-HTML
95 &copy; L&Ouml;VE Development Team<br>39 &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>
96 Licensed under the GNU Free Documentation License, Version 1.3.40 Licensed under the GNU Free Documentation License, Version 1.3.
97 HTML41 HTML
98 end42 end
public/icons/docs/love/16.png+0−0

Binary file not shown.

public/icons/docs/love/16@2x.png+0−0

Binary file not shown.