History

Finish LOVE scraper

Thibaut Courouble committed 10 years ago

Commit e3582e21 parent a25290d12 files · +117 −212Browse files

Unified · Split

12 files
  1. assets/images/icons.png +0 −0
  2. assets/images/icons@2x.png +0 −0
  3. assets/javascripts/news.json +4 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/stylesheets/global/_icons.scss +1 −0
  6. assets/stylesheets/pages/_love.scss +9 −59
  7. lib/docs/filters/core/internal_urls.rb +1 −0
  8. lib/docs/filters/love/clean_html.rb +37 −70
  9. lib/docs/filters/love/entries.rb +47 −13
  10. lib/docs/scrapers/love.rb +17 −73
  11. public/icons/docs/love/16.png +0 −0
  12. public/icons/docs/love/16@2x.png +0 −0
assets/images/icons.png+0−0

Binary file not shown.

assets/images/icons@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+4−1
@@ -1,10 +1,13 @@
11 [
22 [
3+ "2016-06-19",
4+ "New documentation: <a href=\"/love/\">L&Ouml;VE</a>"
5+ ], [
36 "2016-06-12",
47 "New documentation: <a href=\"/angular/\">Angular 2</a>"
58 ], [
69 "2016-06-05",
7− "New documentation: <a href=\"/kotlin/\">Kotlin</a> and <a href=\"/padrino/\">Padrino</a>"
10+ "New documentations: <a href=\"/kotlin/\">Kotlin</a> and <a href=\"/padrino/\">Padrino</a>"
811 ], [
912 "2016-04-24",
1013 "New documentations: <a href=\"/numpy/\">NumPy</a> and <a href=\"/apache_pig/\">Apache Pig</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -275,6 +275,11 @@ credits = [ · 274 unmodified lines
275275 '1994–2015 Lua.org, PUC-Rio',
276276 'MIT',
277277 'http://www.lua.org/license.html'
278+ ], [
279+ 'L&Ouml;VE',
280+ '2006-2016 L&Ouml;VE Development Team',
281+ 'GFDL',
282+ 'http://www.gnu.org/copyleft/fdl.html'
278283 ], [
279284 'Marionette.js',
280285 '2016 Muted Solutions, LLC',
assets/stylesheets/global/_icons.scss+1−0
@@ -142,3 +142,4 @@ · 141 unmodified lines
142142 ._icon-kotlin:before { background-position: -6rem -11rem; }
143143 ._icon-padrino:before { background-position: -7rem -11rem; }
144144 ._icon-angular:before { background-position: -8rem -11rem; }
145+._icon-love:before { background-position: -9rem -11rem; }
assets/stylesheets/pages/_love.scss+9−59
@@ -1,67 +1,17 @@
11 ._love {
2− padding-left: 1rem;
2+ @extend %simple;
33
4− h1, h2 { margin-left: -1rem; }
5− h2 { @extend %block-heading; }
6− h3 { margin-left: -0.5rem; @extend %block-label; }
7−
8− ._mobile & {
9− padding-left: 0;
10−
11− h1, h2, h3 { margin-left: 0; }
12− }
13−
14− p > code, li > code { @extend %label; }
15− blockquote { @extend %note; }
16−
17− .box { @extend %box; }
184 .note { @extend %note; }
19− .label { @extend %label; }
5+ .note-green { @extend %note-green; }
6+ .note-red { @extend %note-red; }
207
21− .box-heading {
22− @extend %heading-box;
23− padding: .5em .75em;
24− margin-top: 1.5rem;
25− margin-bottom: 0px;
26− border-bottom: none;
27− border-bottom-left-radius: 0px;
28− border-bottom-right-radius: 0px;
29− }
30− .box-with-heading {
31− @extend %box;
32− padding: .5em .75em;
33− margin-top: 0px;
34− margin-bottom: 1.5rem;
35− border-top-left-radius: 0px;
36− border-top-right-radius: 0px;
37− }
8+ .label, dt > code { @extend %label; }
9+ .label-green { @extend %label-green; }
10+ .label-red { @extend %label-red; }
11+
12+ .smwtable { width: 100%; }
13+ .smwtable td:nth-last-child(2), .smwtable td:last-child { width: 2.5em; }
3814
39− .note-green, .label-green { @extend %note-green; }
40− .note-red, .label-red { @extend %note-red; }
41− .note-orange, .label-orange { @extend %note-orange; }
4215 .cell-green { background: $noteGreenBackground; }
4316 .cell-red { background: $noteRedBackground; }
44−
45− .smwtable {
46− width: 100%;
47− tr {
48− td {
49− word-wrap: break-word;
50− }
51−
52− td:first-child, td:nth-last-child(2), td:last-child {
53− vertical-align: middle;
54− white-space: nowrap;
55− width: 1em;
56− overflow: hidden;
57− }
58− }
59− }
60−
61− hr {
62− border: none;
63− height: 1px;
64− background-color: $textColorLighter;
65− margin: 1.5em 0 1em;
66− }
6717 }
lib/docs/filters/core/internal_urls.rb+1−0
@@ -39,6 +39,7 @@ module Docs · 38 unmodified lines
3939 def to_internal_url(str)
4040 return unless (url = parse_url(str)) && (subpath = subpath_to(url))
4141 normalize_subpath(subpath)
42+ subpath = URI.unescape(subpath) if context[:decode_and_clean_paths]
4243 return if skip_subpath?(subpath)
4344 normalize_url(url, subpath)
4445 url
lib/docs/filters/love/clean_html.rb+37−70
@@ -2,100 +2,67 @@ module Docs · 1 unmodified line
22 class Love
33 class CleanHtmlFilter < Filter
44 def call
5− # Fix syntax highlighting
5+ @doc = at_css('#mw-content-text')
6+
67 css('.mw-code').each do |node|
7− node.content = node.at_css("div > pre").content
8+ node.content = node.at_css('div > pre').content
89 node['data-language'] = 'lua'
910 node.name = 'pre'
1011 end
1112
12− # Move header tags up
13− css('h2', 'h3').each do |node|
14− headline = node.at_css('.mw-headline')
15− node['id'] = headline['id']
16− node.content = headline.inner_text
13+ css('span[id]').each do |node|
14+ node.parent['id'] = node['id']
15+ node.before(node.children).remove
1716 end
1817
19− # Move dt tags up
20− css('dt > span').each do |node|
21− node.parent.content = node.inner_text
18+ css('table.notice').each do |node|
19+ content = node.at_css('td:nth-child(2)').inner_html
20+ node.replace %(<p class="note">#{content}</p>)
2221 end
2322
24− # Style notices and new/removed sections
25− css('.notice', '.new-section', '.removed-section', '.removed-new-section').each do |node|
26− case node['class']
27− when 'notice'
28− node['class'] = 'note note-warning'
29− node.inner_html = node.at_css('td:nth-child(2)').inner_html
30− node.next.remove unless node.next.nil? or node.next.name != 'br'
31− when 'new-section', 'removed-section', 'removed-new-section'
32− node['class'] = node['class'] == 'new-section' ? 'note note-green' : 'note note-red'
33− node.inner_html = node.at_css('tr > td > i').inner_html \
34− + '<br>' \
35− + node.at_css('tr > td > small').inner_html
36− end
37−
38− node.name = 'p'
39− node.remove_attribute('bgcolor')
40− node.remove_attribute('style')
41− node.remove_attribute('align')
23+ css('table.new-section', 'table.removed-section', 'table.removed-new-section').each do |node|
24+ klass = node['class'] == 'new-section' ? 'note-green' : 'note-red'
25+ content = node.css('td').map(&:inner_html).join('<br>')
26+ node.replace %(<p class="note #{klass}">#{content}</p>)
4227 end
4328
44− # Style new/removed features
4529 css('.new-feature', '.removed-feature', '.removed-new-feature').each do |node|
46− node.name = 'div'
47− node['class'] = node['class'] == 'new-feature' ? 'box-heading label-green' : 'box-heading label-red'
48− node.remove_attribute('style')
30+ klass = node['class'] == 'new-feature' ? 'label-green' : 'label-red'
31+ content = node.content.sub(' LÖVE', '')
32+ label = %( <span class="label #{klass}">#{content}</span>)
4933
50− container = node.next_element
51− container.name = 'div'
52− container['class'] = 'box-with-heading'
53− container.remove_attribute('style')
34+ node.next_element.css('dt').each { |n| n << label }
35+ node.remove
5436 end
5537
56− # Style tables
57− css('table.smwtable').each do |table|
58− table.remove_attribute('style')
59− table.css('td').each do |cell|
60− cell.remove_attribute('style')
61− end
62− table.css('td:last-child', 'td:nth-last-child(2)').each do |cell|
63− img = cell.at_css('img')
64− if img then
65− if img['alt'] == 'Added since' then
66− cell['class'] = 'cell-green'
67− elsif img['alt'] == 'Removed in'
68− cell['class'] = 'cell-red'
69− end
70− img.remove
71− end
72− end
38+ css('img[src$="Add.png"]').each do |node|
39+ node.parent['class'] = 'cell-green'
40+ node.remove
7341 end
7442
75− # Remove Other Languages
76− css('#Other_Languages').remove
77− css('.i18n').remove
43+ css('img[src$="Remove.png"]').each do |node|
44+ node.parent['class'] = 'cell-red'
45+ node.remove
46+ end
7847
79− # Remove changelog
80− node = at_css('h2#Changelog')
81− if !node.nil? then
82− begin
83− nxt = node.next
84− node.remove
85− node = nxt
86− end while !node.nil? and node.name != 'h2'
48+ css('table, tr, td, th').each do |node|
49+ %w(style cellpadding cellspacing width height valign).each do |attribute|
50+ node.remove_attribute(attribute)
51+ end
8752 end
8853
89− # Remove empty paragraphs
90− css('p').each do |node|
91− node.remove if node.inner_text.strip == ''
54+ css('.note i', '.note small', 'div:not([class])', '.smwtable td:nth-last-child(2) > a', '.smwtable td:last-child > a').each do |node|
55+ node.before(node.children).remove
9256 end
9357
94− # Remove linebreaks that are the first or last child of a paragraph
95− css('p > br:first-child', 'p > br:last-child').each do |node|
96− node.remove
58+ css('p > br').each do |node|
59+ node.parent.remove if node.parent.content.empty?
9760 end
9861
62+ css('div > br', '> br', 'hr').remove
63+ css('#Editing_the_wiki + p', '#Editing_the_wiki').remove
64+ css('#Other_Languages', '.i18n').remove
65+
9966 doc
10067 end
10168 end
lib/docs/filters/love/entries.rb+47−13
@@ -1,21 +1,55 @@
11 module Docs
22 class Love
33 class EntriesFilter < Docs::EntriesFilter
4+ TYPES = {
5+ 'require' => 'Lua',
6+ 'light_userdata' => 'Lua',
7+ 'value' => 'Lua',
8+ 'variable' => 'Lua',
9+
10+ 'Audio_Formats' => 'love.sound',
11+ 'ImageFontFormat' => 'love.font',
12+ 'BlendMode_Formulas' => 'love.graphics',
13+ 'Shader_Variables' => 'love.graphics',
14+
15+ 'AreaSpreadDistribution' => 'love.graphics',
16+ 'BodyType' => 'love.physics',
17+ 'BufferMode' => 'love.filesystem',
18+ 'CompressedFormat' => 'love.image',
19+ 'JoystickConstant' => 'love.joystick',
20+ 'ParticleInsertMode' => 'love.graphics',
21+ 'String' => 'love',
22+ 'TextureMode' => 'love.graphics'
23+ }
24+
25+ def call
26+ if context[:initial_paths].include?(slug)
27+ css('table.smwtable td:first-child > a').each do |node|
28+ TYPES[node.content.strip] = slug
29+ end
30+ end
31+
32+ super
33+ end
34+
435 def get_type
5− if TYPE_OVERRIDE.key?(slug) then
6− return TYPE_OVERRIDE[slug]
7− elsif m = slug.match(/\A(love\.\w+)\z/) then
8− # modules and funcions
9− return LOVE_MODULES.include?(m[1]) ? m[1] : 'love'
10− elsif m = slug.match(/\A(love\.\w+)\.(\w+)/) then
11− # functions in modules
12− return m[1]
13− elsif context[:list_classes] and (m = slug.match(/\A\(?([A-Z]\w+)\)?(\:\w+)?/)) then
14− # classes, their members and enums
15− return m[1] unless m[1].include?('_')
36+ if slug == 'love'
37+ 'love'
38+ elsif slug.start_with?('enet')
39+ 'enet'
40+ elsif slug.include?('Joint') || slug.include?('Shape')
41+ 'love.physics'
42+ elsif TYPES.key?(slug)
43+ TYPES[slug]
44+ elsif match = slug.match(/\A(love\.\w+)(\.\w+)?\z/)
45+ match[2] || context[:initial_paths].include?(match[1]) ? match[1] : 'love'
46+ elsif at_css('#catlinks a[title="Category:Lua"]')
47+ 'Lua'
48+ elsif
49+ type = slug.split(':').first
50+ type.remove! %r{[\(\)]}
51+ TYPES[type]
1652 end
17− # usually this shouldn't happen
18− "Other"
1953 end
2054 end
2155 end
lib/docs/scrapers/love.rb+17−73
@@ -1,98 +1,42 @@
11 module Docs
22 class Love < UrlScraper
3− LOVE_MODULES = %w(
4− love
5− love.audio
6− love.event
7− love.filesystem
8− love.font
9− love.graphics
10− love.image
11− love.joystick
12− love.keyboard
13− love.math
14− love.mouse
15− love.physics
16− love.sound
17− love.system
18− love.thread
19− love.timer
20− love.touch
21− love.video
22− love.window
23− )
24− TYPE_OVERRIDE = {
25− "Audio_Formats" => "love.sound",
26− "ImageFontFormat" => "love.font",
27− "BlendMode_Formulas" => "BlendMode",
28− "Shader_Variables" => "Shader"
29− }
30−
313 self.name = 'LÖVE'
324 self.slug = 'love'
335 self.type = 'love'
6+ self.release = '0.10.1'
347 self.base_url = 'https://love2d.org/wiki/'
35− self.root_path = 'love'
36− self.initial_paths = LOVE_MODULES
8+ self.root_path = 'Main_Page'
9+ self.initial_paths = %w(love love.audio love.event love.filesystem love.font love.graphics
10+ love.image love.joystick love.keyboard love.math love.mouse love.physics love.sound
11+ love.system love.thread love.timer love.touch love.video love.window enet socket utf8)
3712 self.links = {
3813 home: 'https://love2d.org/',
3914 code: 'https://bitbucket.org/rude/love'
4015 }
4116
42− html_filters.push 'love/clean_html', 'love/entries', 'title'
43−
44− options[:root_title] = 'love'
45− options[:initial_paths] = LOVE_MODULES
17+ html_filters.push 'love/entries', 'love/clean_html', 'title'
4618
19+ options[:root_title] = 'LÖVE'
4720 options[:decode_and_clean_paths] = true
21+ options[:container] = '#bodyContent'
4822
49− # Add types to classes and their members
50− options[:list_classes] = true
23+ options[:skip] = %w(Getting_Started Building_LÖVE Tutorial Tutorials Game_Distribution License
24+ Games Libraries Software Snippets Version_History Lovers PO2_Syndrome HSL_color Guidelines)
5125
52− options[:container] = '#mw-content-text'
53−
54− options[:only_patterns] = [
55− /\A(love\z|love\.|[A-Z]|\([A-Z])/
56− # love
57− # love.* (modules and functions)
58− # Uppercased (classes and enums)
59− # (Uppercased) (generalized classes)
60− ]
61− options[:skip] = %w(
62− Getting_Started
63− Building_LÖVE
64− Tutorial
65− Tutorials
66− Game_Distribution
67− License
68− Games
69− Libraries
70− Software
71− Snippets
72− Version_History
73− Lovers
74− PO2_Syndrome
75− HSL_color
76− )
7726 options[:skip_patterns] = [
78− /_\([^\)]+\)\z/,
79− # anything_(language) (this might have to be tweaked)
80− /\ASpecial:/,
81− /\ACategory:/,
82− /\AFile:/,
83− /\AHelp:/,
84− /\ATemplate:/,
85− /\AUser:/,
86− /\ATutorial:/
87− # special pages are indistinguishable from instance methods
27+ /_\([^\)]+\)\z/, # anything_(language)
28+ /\A(Special|Category|File|Help|Template|User|Tutorial):/,
29+ /\A\d/
8830 ]
8931
9032 options[:replace_paths] = {
91− "Config_Files" => "love.conf"
33+ 'Config_Files' => 'love.conf',
34+ 'conf.lua' => 'love.conf',
35+ 'lua-enet' => 'enet'
9236 }
9337
9438 options[:attribution] = <<-HTML
95− &copy; L&Ouml;VE Development Team<br>
39+ &copy; 2006&ndash;2016 L&Ouml;VE Development Team<br>
9640 Licensed under the GNU Free Documentation License, Version 1.3.
9741 HTML
9842 end
public/icons/docs/love/16.png+0−0

Binary file not shown.

public/icons/docs/love/16@2x.png+0−0

Binary file not shown.