History

Finish Nim scraper

Thibaut Courouble committed 9 years ago

Commit a4ba32e1 parent 09eea6617 files · +151 −135Browse files

Unified · Split

17 files
  1. assets/images/docs-2.png +0 −0
  2. assets/images/docs-2@2x.png +0 −0
  3. assets/javascripts/news.json +3 −0
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/javascripts/vendor/prism.js +40 −3
  6. assets/stylesheets/application-dark.css.scss +0 −1
  7. assets/stylesheets/application.css.scss +0 −1
  8. assets/stylesheets/components/_prism.scss +6 −1
  9. assets/stylesheets/global/_icons.scss +1 −0
  10. assets/stylesheets/pages/_nim.scss +0 −81
  11. assets/stylesheets/pages/_simple.scss +1 −0
  12. lib/docs/filters/nim/clean_html.rb +46 −12
  13. lib/docs/filters/nim/entries.rb +44 −31
  14. lib/docs/scrapers/nim.rb +8 −8
  15. public/icons/docs/nim/16.png +0 −0
  16. public/icons/docs/nim/16@2x.png +0 −0
  17. public/icons/docs/nim/SOURCE +1 −1
assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
1[1[
2 [2 [
3 "2017-09-03",
4 "New documentation: <a href=\"/nim/\">Nim</a>"
5 ], [
3 "2017-07-23",6 "2017-07-23",
4 "New documentation: <a href=\"/godot/\">Godot</a>"7 "New documentation: <a href=\"/godot/\">Godot</a>"
5 ], [8 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -423,6 +423,11 @@ credits = [ · 422 unmodified lines
423 '2009-2016 Xiaozhe Wang (chaoslawful)<br>&copy; 2009-2017 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.',423 '2009-2016 Xiaozhe Wang (chaoslawful)<br>&copy; 2009-2017 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.',
424 'BSD',424 'BSD',
425 'https://github.com/openresty/lua-nginx-module#copyright-and-license'425 'https://github.com/openresty/lua-nginx-module#copyright-and-license'
426 ], [
427 'Nim',
428 '2006-2017 Andreas Rumpf',
429 'MIT',
430 'https://github.com/nim-lang/Nim#license'
426 ], [431 ], [
427 'Node.js',432 'Node.js',
428 'Joyent, Inc. and other Node contributors<br>Node.js is a trademark of Joyent, Inc.',433 'Joyent, Inc. and other Node contributors<br>Node.js is a trademark of Joyent, Inc.',
assets/javascripts/vendor/prism.js+40−3
@@ -1,4 +1,4 @@
1/* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+c+cpp+coffeescript+ruby+elixir+erlang+go+java+json+kotlin+lua+nginx+perl+php+python+crystal+rust+scss+sql+typescript */1/* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+c+cpp+coffeescript+ruby+elixir+erlang+go+java+json+kotlin+lua+nginx+nim+perl+php+python+crystal+rust+scss+sql+typescript */
2var _self = (typeof window !== 'undefined')2var _self = (typeof window !== 'undefined')
3 ? window // if in browser3 ? window // if in browser
4 : (4 : (
@@ -568,6 +568,9 @@ Prism.languages.markup = { · 563 unmodified lines
568 'entity': /&#?[\da-z]{1,8};/i568 'entity': /&#?[\da-z]{1,8};/i
569};569};
570 570
571Prism.languages.markup['tag'].inside['attr-value'].inside['entity'] =
572 Prism.languages.markup['entity'];
573
571// Plugin to make entity title show the real entity, idea by Roman Komarov574// Plugin to make entity title show the real entity, idea by Roman Komarov
572Prism.hooks.add('wrap', function(env) {575Prism.hooks.add('wrap', function(env) {
573 576
@@ -664,7 +667,7 @@ Prism.languages.clike = { · 90 unmodified lines
664 667
665Prism.languages.javascript = Prism.languages.extend('clike', {668Prism.languages.javascript = Prism.languages.extend('clike', {
666 'keyword': /\b(as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield)\b/,669 'keyword': /\b(as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield)\b/,
667 'number': /\b-?(0x[\dA-Fa-f]+|0b[01]+|0o[0-7]+|\d*\.?\d+([Ee][+-]?\d+)?|NaN|Infinity)\b/,670 'number': /\b-?(0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|\d*\.?\d+([Ee][+-]?\d+)?|NaN|Infinity)\b/,
668 // Allow for all non-ASCII characters (See http://stackoverflow.com/a/2008444)671 // Allow for all non-ASCII characters (See http://stackoverflow.com/a/2008444)
669 'function': /[_$a-zA-Z\xA0-\uFFFF][_$a-zA-Z0-9\xA0-\uFFFF]*(?=\()/i,672 'function': /[_$a-zA-Z\xA0-\uFFFF][_$a-zA-Z0-9\xA0-\uFFFF]*(?=\()/i,
670 'operator': /-[-=]?|\+[+=]?|!=?=?|<<?=?|>>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/=?|~|\^=?|%=?|\?|\.{3}/673 'operator': /-[-=]?|\+[+=]?|!=?=?|<<?=?|>>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/=?|~|\^=?|%=?|\?|\.{3}/
@@ -672,7 +675,7 @@ Prism.languages.javascript = Prism.languages.extend('clike', { · 1 unmodified line
672 675
673Prism.languages.insertBefore('javascript', 'keyword', {676Prism.languages.insertBefore('javascript', 'keyword', {
674 'regex': {677 'regex': {
675 pattern: /(^|[^/])\/(?!\/)(\[.+?]|\\.|[^/\\\r\n])+\/[gimyu]{0,5}(?=\s*($|[\r\n,.;})]))/,678 pattern: /(^|[^/])\/(?!\/)(\[[^\]\r\n]+]|\\.|[^/\\\[\r\n])+\/[gimyu]{0,5}(?=\s*($|[\r\n,.;})]))/,
676 lookbehind: true,679 lookbehind: true,
677 greedy: true680 greedy: true
678 }681 }
@@ -710,6 +713,7 @@ if (Prism.languages.markup) { · 31 unmodified lines
710}713}
711 714
712Prism.languages.js = Prism.languages.javascript;715Prism.languages.js = Prism.languages.javascript;
716
713Prism.languages.c = Prism.languages.extend('clike', {717Prism.languages.c = Prism.languages.extend('clike', {
714 'keyword': /\b(asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while)\b/,718 'keyword': /\b(asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while)\b/,
715 'operator': /\-[>-]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|?\||[~^%?*\/]/,719 'operator': /\-[>-]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|?\||[~^%?*\/]/,
@@ -1245,6 +1249,39 @@ Prism.languages.nginx = Prism.languages.extend('clike', { · 529 unmodified lines
1245Prism.languages.insertBefore('nginx', 'keyword', {1249Prism.languages.insertBefore('nginx', 'keyword', {
1246 'variable': /\$[a-z_]+/i1250 'variable': /\$[a-z_]+/i
1247});1251});
1252Prism.languages.nim = {
1253 'comment': /#.*/,
1254 // Double-quoted strings can be prefixed by an identifier (Generalized raw string literals)
1255 // Character literals are handled specifically to prevent issues with numeric type suffixes
1256 'string': {
1257 pattern: /(?:(?:\b(?!\d)(?:\w|\\x[8-9a-fA-F][0-9a-fA-F])+)?(?:"""[\s\S]*?"""(?!")|"(?:\\[\s\S]|""|[^"\\])*")|'(?:\\(?:\d+|x[\da-fA-F]{2}|.)|[^'])')/,
1258 greedy: true
1259 },
1260 // The negative look ahead prevents wrong highlighting of the .. operator
1261 'number': /\b(?:0[xXoObB][\da-fA-F_]+|\d[\d_]*(?:(?!\.\.)\.[\d_]*)?(?:[eE][+-]?\d[\d_]*)?)(?:'?[iuf]\d*)?/,
1262 'keyword': /\b(?:addr|as|asm|atomic|bind|block|break|case|cast|concept|const|continue|converter|defer|discard|distinct|do|elif|else|end|enum|except|export|finally|for|from|func|generic|if|import|include|interface|iterator|let|macro|method|mixin|nil|object|out|proc|ptr|raise|ref|return|static|template|try|tuple|type|using|var|when|while|with|without|yield)\b/,
1263 'function': {
1264 pattern: /(?:(?!\d)(?:\w|\\x[8-9a-fA-F][0-9a-fA-F])+|`[^`\r\n]+`)\*?(?:\[[^\]]+\])?(?=\s*\()/,
1265 inside: {
1266 'operator': /\*$/
1267 }
1268 },
1269 // We don't want to highlight operators inside backticks
1270 'ignore': {
1271 pattern: /`[^`\r\n]+`/,
1272 inside: {
1273 'punctuation': /`/
1274 }
1275 },
1276 'operator': {
1277 // Look behind and look ahead prevent wrong highlighting of punctuations [. .] {. .} (. .)
1278 // but allow the slice operator .. to take precedence over them
1279 // One can define his own operators in Nim so all combination of operators might be an operator.
1280 pattern: /(^|[({\[](?=\.\.)|(?![({\[]\.).)(?:(?:[=+\-*\/<>@$~&%|!?^:\\]|\.\.|\.(?![)}\]]))+|\b(?:and|div|of|or|in|is|isnot|mod|not|notin|shl|shr|xor)\b)/m,
1281 lookbehind: true
1282 },
1283 'punctuation': /[({\[]\.|\.[)}\]]|[`(){}\[\],:]/
1284};
1248Prism.languages.perl = {1285Prism.languages.perl = {
1249 'comment': [1286 'comment': [
1250 {1287 {
assets/stylesheets/application-dark.css.scss+0−1
@@ -67,7 +67,6 @@ · 66 unmodified lines
67 'pages/meteor',67 'pages/meteor',
68 'pages/modernizr',68 'pages/modernizr',
69 'pages/moment',69 'pages/moment',
70 'pages/nim',
71 'pages/nginx',70 'pages/nginx',
72 'pages/node',71 'pages/node',
73 'pages/npm',72 'pages/npm',
assets/stylesheets/application.css.scss+0−1
@@ -67,7 +67,6 @@ · 66 unmodified lines
67 'pages/meteor',67 'pages/meteor',
68 'pages/modernizr',68 'pages/modernizr',
69 'pages/moment',69 'pages/moment',
70 'pages/nim',
71 'pages/nginx',70 'pages/nginx',
72 'pages/node',71 'pages/node',
73 'pages/npm',72 'pages/npm',
assets/stylesheets/components/_prism.scss+6−1
@@ -77,6 +77,11 @@ · 76 unmodified lines
77 }77 }
78}78}
79 79
80.token.important {80.token.important,
81.token.bold {
81 font-weight: $boldFontWeight;82 font-weight: $boldFontWeight;
82}83}
84
85.token.italic {
86 font-style: italic;
87}
assets/stylesheets/global/_icons.scss+1−0
@@ -173,3 +173,4 @@ · 172 unmodified lines
173._icon-electron:before { background-position: -2rem -2rem; @extend %doc-icon-2; }173._icon-electron:before { background-position: -2rem -2rem; @extend %doc-icon-2; }
174._icon-falcon:before { background-position: -3rem -2rem; @extend %doc-icon-2; }174._icon-falcon:before { background-position: -3rem -2rem; @extend %doc-icon-2; }
175._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }175._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }
176._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
assets/stylesheets/pages/_nim.scss+0−81
@@ -1,81 +0,0 @@
1._nim {
2 @extend %simple;
3
4 @if $style == 'dark' {
5 span.DecNumber { color: #AE81FF; }
6 span.BinNumber { color: #AE81FF; }
7 span.HexNumber { color: #AE81FF; }
8 span.OctNumber { color: #AE81FF; }
9 span.FloatNumber { color: #AE81FF; }
10 span.Identifier { color: #F8F8F2; }
11 span.Keyword { font-weight: 600; color: #F92672; }
12 span.StringLit { color: #E6DB74; }
13 span.LongStringLit { color: #E6DB74; }
14 span.CharLit { color: #E6DB74; }
15 span.EscapeSequence { color: white; }
16 span.Operator { color: white; }
17 span.Punctuation {color: white; }
18 span.Comment, span.LongComment {
19 font-style: italic;
20 font-weight: 400;
21 color: #75715E; }
22
23 span.RegularExpression { color: darkviolet; }
24 span.TagStart { color: #F92672; }
25 span.TagEnd { color: #F92672; }
26 span.Key { color: #AE81FF; }
27 span.Value { color: #AE81FF; }
28 span.RawData { color: #a4255b; }
29 span.Assembler { color: #AE81FF; }
30 span.Preprocessor { color: #AE81FF; }
31 span.Directive { color: #AE81FF; }
32
33 span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
34 span.Other { color: white; }
35
36 /* Pop type, const, proc, and iterator defs in nim def blocks */
37 dt pre > span.Identifier, dt pre > span.Operator { color: #529B2F; font-weight: 700; }
38 } @else {
39 span.DecNumber { color: #252dbe; }
40 span.BinNumber { color: #252dbe; }
41 span.HexNumber { color: #252dbe; }
42 span.OctNumber { color: #252dbe; }
43 span.FloatNumber { color: #252dbe; }
44 span.Identifier { color: #3b3b3b; }
45 span.Keyword { font-weight: 600; color: #5e8f60; }
46 span.StringLit { color: #a4255b; }
47 span.LongStringLit { color: #a4255b; }
48 span.CharLit { color: #a4255b; }
49 span.EscapeSequence { color: black; }
50 span.Operator { color: black; }
51 span.Punctuation {color: black; }
52 span.Comment, span.LongComment {
53 font-style: italic;
54 font-weight: 400;
55 color: #484a86; }
56
57 span.RegularExpression { color: darkviolet; }
58 span.TagStart { color: darkviolet; }
59 span.TagEnd { color: darkviolet; }
60 span.Key { color: #252dbe; }
61 span.Value { color: #252dbe; }
62 span.RawData { color: #a4255b; }
63 span.Assembler { color: #252dbe; }
64 span.Preprocessor { color: #252dbe; }
65 span.Directive { color: #252dbe; }
66
67 span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
68 span.Other { color: black; }
69
70 /* Pop type, const, proc, and iterator defs in nim def blocks */
71 dt pre > span.Identifier, dt pre > span.Operator { color: #155da4; font-weight: 700; }
72 }
73 dt pre > span.Identifier ~ span.Identifier, dt pre > span.Operator ~ span.Identifier {
74 color: inherit;
75 font-weight: inherit; }
76
77 dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator {
78 color: inherit;
79 font-weight: inherit; }
80
81}
assets/stylesheets/pages/_simple.scss+1−0
@@ -38,6 +38,7 @@ · 37 unmodified lines
38._markdown,38._markdown,
39._mocha,39._mocha,
40._mongoose,40._mongoose,
41._nim,
41._redux,42._redux,
42._requirejs,43._requirejs,
43._typescript,44._typescript,
lib/docs/filters/nim/clean_html.rb+46−12
@@ -4,26 +4,60 @@ module Docs · 3 unmodified lines
4 def call4 def call
5 @doc = at_css('#documentId .container')5 @doc = at_css('#documentId .container')
6 6
7 css('.docinfo').remove7 css('.docinfo', '.footer', 'blockquote > p:empty', '.link-seesrc').remove
8 8
9 content = at_css('#content')9 css('h1:not(.title), h2, h3, h4').each do |node|
10 if content != nil10 node.name = node.name.sub(/\d/) { |i| i.to_i + 1 }
11 at_css('#content').remove_attribute('class')
12 @doc.add_child(at_css('#content').inner_html)
13 end11 end
14 12
15 css('> div.row').remove13 if content = at_css('#content')
14 content.prepend_child at_css('h1.title')
15 @doc = content
16 end
16 17
17 css('pre').each do |node|18 if root_page?
18 node['data-language'] = 'nim'19 at_css('h1').content = 'Nim Documentation'
19 end20 end
20 21
21 # remove link from headers
22 css('h1 > a', 'h2 > a', 'h3 > a', 'h4 > a').each do |node|22 css('h1 > a', 'h2 > a', 'h3 > a', 'h4 > a').each do |node|
23 node.parent['id'] = node['id']23 node.parent['id'] = node['id'] if node['id']
24 node.parent.content = node.content24 node.before(node.children).remove
25 end
26
27 css('a[name]').each do |node|
28 node.next_element['id'] = node['name']
29 node.remove
30 end
31
32 css('pre').each do |node|
33 node.content = node.content.strip
34 node['data-language'] = 'nim' unless node.content =~ /\A[\w\-\_\:\=\ ]+\z/
25 end35 end
26 36
37 css('tt').each do |node|
38 node.name = 'code'
39 end
40
41 css('cite').each do |node|
42 node.name = 'em'
43 end
44
45 css('.section').each do |node|
46 node.first_element_child['id'] = node['id'] if node['id']
47 node.before(node.children).remove
48 end
49
50 css('span.pre').each do |node|
51 node.before(node.children).remove
52 end
53
54 css('blockquote > pre:only-child', 'blockquote > dl:only-child', 'blockquote > table').each do |node|
55 node.parent.before(node.parent.children).remove
56 end
57
58 css('a', 'dl', 'table', 'code').remove_attr('class')
59 css('table').remove_attr('border')
60
27 doc61 doc
28 end62 end
29 end63 end
lib/docs/filters/nim/entries.rb+44−31
@@ -1,49 +1,62 @@
1module Docs1module Docs
2 class Nim2 class Nim
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_type4 def get_name
5 at_css('h1').content5 name = at_css('h1').content
6 name.remove! 'Module '
7 name.remove! ' User Guide'
8 name.remove! ' User\'s manual'
9 name.remove! %r{ \-.*}
10 name.strip!
11 name
6 end12 end
7 13
8 def get_name14 def get_type
9 at_css('h1').content15 if name.include?('Tutorial')
16 'Tutorial'
17 elsif slug == 'manual'
18 'Manual'
19 elsif at_css('h1').content.include?('Module ')
20 name
21 else
22 'Reference'
23 end
10 end24 end
11 25
12 def additional_entries26 def additional_entries
13 entries = []27 entries = []
14 if get_name.start_with? 'Module '28
15 module_name = get_name[7..-1]29 if at_css('h1').content.include?('Module ')
16 css('div .section').map do |node|30 css('#toc-list > li > .simple-toc-section').each do |node|
17 section_node = node.at_css('h1 a')31 type = node.previous_element.content.strip
18 if section_node != nil32
19 section_name = section_node.content.strip33 node.css('a.reference:not(.reference-toplevel)').each do |n|
20 items_node = node.at_css('dl.item')34 n.css('span').remove
21 if items_node != nil35 name = n.content.strip
22 items_node.css('dt a').map do |item_node|36 name << '()' if (type == 'Procs' || type == 'Templates') && !name.include?('`')
23 item_name = item_node['name']37 name.remove! '`'
24 if item_name.include? ','38 name.prepend "#{self.name}."
25 item_name = item_name.sub(',', '(') + ')'39 id = n['href'].remove('#')
26 end40 entries << [name, id] unless entries.any? { |e| e[0] == name }
27 entries << [module_name + '.' + item_name, item_node.parent['id']]
28 end
29 end
30 end41 end
31 end42 end
32 else43 elsif slug == 'manual'
33 css('h1', 'h2', 'h3').map do |node|44 css('#toc-list > li > a').each do |node|
34 id = node['id']
35 name = node.content.strip45 name = node.content.strip
36 if id != nil46 next if name.start_with?('About')
37 entries << [name, id]47 id = node['href'].remove('#')
38 else48 entries << [name, id]
39 a = node.at_css('a')49 end
40 if a != nil50
41 id = a['id']51 css('#toc-list > ul').each do |node|
42 entries << [name, id]52 type = node.previous_element.content.strip
43 end53
54 node.css('> li > a').each do |n|
55 entries << [n.content.strip, n['href'].remove('#'), "Manual: #{type}"]
44 end56 end
45 end57 end
46 end58 end
59
47 entries60 entries
48 end61 end
49 end62 end
lib/docs/scrapers/nim.rb+8−8
@@ -2,20 +2,20 @@ module Docs · 1 unmodified line
2 class Nim < UrlScraper2 class Nim < UrlScraper
3 self.type = 'nim'3 self.type = 'nim'
4 self.release = '0.17.0'4 self.release = '0.17.0'
5 self.base_url = 'https://nim-lang.org/docs/'
6 self.root_path = 'overview.html'
5 self.links = {7 self.links = {
6 home: 'https://nim-lang.org/',8 home: 'https://nim-lang.org/',
7 code: 'https://github.com/nim-lang/Nim'9 code: 'https://github.com/nim-lang/Nim'
8 }10 }
9 self.base_url = 'https://nim-lang.org/'
10 self.root_path = 'docs/overview.html'
11 11
12 html_filters.push 'nim/entries', 'nim/clean_html' 12 html_filters.push 'nim/entries', 'nim/clean_html'
13 13
14 options[:skip] = %w(cdn-cgi/l/email-protection docs/theindex.html docs/docgen.txt)14 options[:skip] = %w(theindex.html docgen.txt)
15
15 options[:attribution] = <<-HTML16 options[:attribution] = <<-HTML
16 &copy; 2006&ndash;2017 Andreas Rumpf<br>17 &copy; 2006&ndash;2017 Andreas Rumpf<br>
17 All rights reserved. Licensed under the MIT License.18 Licensed under the MIT License.
18 HTML19 HTML
19
20 end20 end
21end21end
public/icons/docs/nim/16.png+0−0

Binary file not shown.

public/icons/docs/nim/16@2x.png+0−0

Binary file not shown.

public/icons/docs/nim/SOURCE+1−1
@@ -1 +1 @@
1https://nim-lang.org/assets/img/logo.svg1https://github.com/nim-lang/website/tree/master/jekyll/assets/img