History

Finish Nim scraper

Thibaut Courouble committed 9 years ago

Commit a4ba32e1 parent 09eea6617 files · +151 −135Browse files

Unified · Split

17 files
  1. assets/images/docs-2.png +0 −0
  2. assets/images/docs-2@2x.png +0 −0
  3. assets/javascripts/news.json +3 −0
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/javascripts/vendor/prism.js +40 −3
  6. assets/stylesheets/application-dark.css.scss +0 −1
  7. assets/stylesheets/application.css.scss +0 −1
  8. assets/stylesheets/components/_prism.scss +6 −1
  9. assets/stylesheets/global/_icons.scss +1 −0
  10. assets/stylesheets/pages/_nim.scss +0 −81
  11. assets/stylesheets/pages/_simple.scss +1 −0
  12. lib/docs/filters/nim/clean_html.rb +46 −12
  13. lib/docs/filters/nim/entries.rb +44 −31
  14. lib/docs/scrapers/nim.rb +8 −8
  15. public/icons/docs/nim/16.png +0 −0
  16. public/icons/docs/nim/16@2x.png +0 −0
  17. public/icons/docs/nim/SOURCE +1 −1
assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+3−0
@@ -1,5 +1,8 @@
11 [
22 [
3+ "2017-09-03",
4+ "New documentation: <a href=\"/nim/\">Nim</a>"
5+ ], [
36 "2017-07-23",
47 "New documentation: <a href=\"/godot/\">Godot</a>"
58 ], [
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -423,6 +423,11 @@ credits = [ · 422 unmodified lines
423423 '2009-2016 Xiaozhe Wang (chaoslawful)<br>&copy; 2009-2017 Yichun "agentzh" Zhang (章亦春), OpenResty Inc.',
424424 'BSD',
425425 'https://github.com/openresty/lua-nginx-module#copyright-and-license'
426+ ], [
427+ 'Nim',
428+ '2006-2017 Andreas Rumpf',
429+ 'MIT',
430+ 'https://github.com/nim-lang/Nim#license'
426431 ], [
427432 'Node.js',
428433 'Joyent, Inc. and other Node contributors<br>Node.js is a trademark of Joyent, Inc.',
assets/javascripts/vendor/prism.js+40−3
@@ -1,4 +1,4 @@
1−/* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+c+cpp+coffeescript+ruby+elixir+erlang+go+java+json+kotlin+lua+nginx+perl+php+python+crystal+rust+scss+sql+typescript */
1+/* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+c+cpp+coffeescript+ruby+elixir+erlang+go+java+json+kotlin+lua+nginx+nim+perl+php+python+crystal+rust+scss+sql+typescript */
22 var _self = (typeof window !== 'undefined')
33 ? window // if in browser
44 : (
@@ -568,6 +568,9 @@ Prism.languages.markup = { · 563 unmodified lines
568568 'entity': /&#?[\da-z]{1,8};/i
569569 };
570570
571+Prism.languages.markup['tag'].inside['attr-value'].inside['entity'] =
572+ Prism.languages.markup['entity'];
573+
571574 // Plugin to make entity title show the real entity, idea by Roman Komarov
572575 Prism.hooks.add('wrap', function(env) {
573576
@@ -664,7 +667,7 @@ Prism.languages.clike = { · 90 unmodified lines
664667
665668 Prism.languages.javascript = Prism.languages.extend('clike', {
666669 'keyword': /\b(as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|var|void|while|with|yield)\b/,
667− 'number': /\b-?(0x[\dA-Fa-f]+|0b[01]+|0o[0-7]+|\d*\.?\d+([Ee][+-]?\d+)?|NaN|Infinity)\b/,
670+ 'number': /\b-?(0[xX][\dA-Fa-f]+|0[bB][01]+|0[oO][0-7]+|\d*\.?\d+([Ee][+-]?\d+)?|NaN|Infinity)\b/,
668671 // Allow for all non-ASCII characters (See http://stackoverflow.com/a/2008444)
669672 'function': /[_$a-zA-Z\xA0-\uFFFF][_$a-zA-Z0-9\xA0-\uFFFF]*(?=\()/i,
670673 'operator': /-[-=]?|\+[+=]?|!=?=?|<<?=?|>>?>?=?|=(?:==?|>)?|&[&=]?|\|[|=]?|\*\*?=?|\/=?|~|\^=?|%=?|\?|\.{3}/
@@ -672,7 +675,7 @@ Prism.languages.javascript = Prism.languages.extend('clike', { · 1 unmodified line
672675
673676 Prism.languages.insertBefore('javascript', 'keyword', {
674677 'regex': {
675− pattern: /(^|[^/])\/(?!\/)(\[.+?]|\\.|[^/\\\r\n])+\/[gimyu]{0,5}(?=\s*($|[\r\n,.;})]))/,
678+ pattern: /(^|[^/])\/(?!\/)(\[[^\]\r\n]+]|\\.|[^/\\\[\r\n])+\/[gimyu]{0,5}(?=\s*($|[\r\n,.;})]))/,
676679 lookbehind: true,
677680 greedy: true
678681 }
@@ -710,6 +713,7 @@ if (Prism.languages.markup) { · 31 unmodified lines
710713 }
711714
712715 Prism.languages.js = Prism.languages.javascript;
716+
713717 Prism.languages.c = Prism.languages.extend('clike', {
714718 'keyword': /\b(asm|typeof|inline|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|union|unsigned|void|volatile|while)\b/,
715719 'operator': /\-[>-]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|?\||[~^%?*\/]/,
@@ -1245,6 +1249,39 @@ Prism.languages.nginx = Prism.languages.extend('clike', { · 529 unmodified lines
12451249 Prism.languages.insertBefore('nginx', 'keyword', {
12461250 'variable': /\$[a-z_]+/i
12471251 });
1252+Prism.languages.nim = {
1253+ 'comment': /#.*/,
1254+ // Double-quoted strings can be prefixed by an identifier (Generalized raw string literals)
1255+ // Character literals are handled specifically to prevent issues with numeric type suffixes
1256+ 'string': {
1257+ pattern: /(?:(?:\b(?!\d)(?:\w|\\x[8-9a-fA-F][0-9a-fA-F])+)?(?:"""[\s\S]*?"""(?!")|"(?:\\[\s\S]|""|[^"\\])*")|'(?:\\(?:\d+|x[\da-fA-F]{2}|.)|[^'])')/,
1258+ greedy: true
1259+ },
1260+ // The negative look ahead prevents wrong highlighting of the .. operator
1261+ 'number': /\b(?:0[xXoObB][\da-fA-F_]+|\d[\d_]*(?:(?!\.\.)\.[\d_]*)?(?:[eE][+-]?\d[\d_]*)?)(?:'?[iuf]\d*)?/,
1262+ 'keyword': /\b(?:addr|as|asm|atomic|bind|block|break|case|cast|concept|const|continue|converter|defer|discard|distinct|do|elif|else|end|enum|except|export|finally|for|from|func|generic|if|import|include|interface|iterator|let|macro|method|mixin|nil|object|out|proc|ptr|raise|ref|return|static|template|try|tuple|type|using|var|when|while|with|without|yield)\b/,
1263+ 'function': {
1264+ pattern: /(?:(?!\d)(?:\w|\\x[8-9a-fA-F][0-9a-fA-F])+|`[^`\r\n]+`)\*?(?:\[[^\]]+\])?(?=\s*\()/,
1265+ inside: {
1266+ 'operator': /\*$/
1267+ }
1268+ },
1269+ // We don't want to highlight operators inside backticks
1270+ 'ignore': {
1271+ pattern: /`[^`\r\n]+`/,
1272+ inside: {
1273+ 'punctuation': /`/
1274+ }
1275+ },
1276+ 'operator': {
1277+ // Look behind and look ahead prevent wrong highlighting of punctuations [. .] {. .} (. .)
1278+ // but allow the slice operator .. to take precedence over them
1279+ // One can define his own operators in Nim so all combination of operators might be an operator.
1280+ pattern: /(^|[({\[](?=\.\.)|(?![({\[]\.).)(?:(?:[=+\-*\/<>@$~&%|!?^:\\]|\.\.|\.(?![)}\]]))+|\b(?:and|div|of|or|in|is|isnot|mod|not|notin|shl|shr|xor)\b)/m,
1281+ lookbehind: true
1282+ },
1283+ 'punctuation': /[({\[]\.|\.[)}\]]|[`(){}\[\],:]/
1284+};
12481285 Prism.languages.perl = {
12491286 'comment': [
12501287 {
assets/stylesheets/application-dark.css.scss+0−1
@@ -67,7 +67,6 @@ · 66 unmodified lines
6767 'pages/meteor',
6868 'pages/modernizr',
6969 'pages/moment',
70− 'pages/nim',
7170 'pages/nginx',
7271 'pages/node',
7372 'pages/npm',
assets/stylesheets/application.css.scss+0−1
@@ -67,7 +67,6 @@ · 66 unmodified lines
6767 'pages/meteor',
6868 'pages/modernizr',
6969 'pages/moment',
70− 'pages/nim',
7170 'pages/nginx',
7271 'pages/node',
7372 'pages/npm',
assets/stylesheets/components/_prism.scss+6−1
@@ -77,6 +77,11 @@ · 76 unmodified lines
7777 }
7878 }
7979
80−.token.important {
80+.token.important,
81+.token.bold {
8182 font-weight: $boldFontWeight;
8283 }
84+
85+.token.italic {
86+ font-style: italic;
87+}
assets/stylesheets/global/_icons.scss+1−0
@@ -173,3 +173,4 @@ · 172 unmodified lines
173173 ._icon-electron:before { background-position: -2rem -2rem; @extend %doc-icon-2; }
174174 ._icon-falcon:before { background-position: -3rem -2rem; @extend %doc-icon-2; }
175175 ._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }
176+._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
assets/stylesheets/pages/_nim.scss+0−81
@@ -1,81 +0,0 @@
1−._nim {
2− @extend %simple;
3−
4− @if $style == 'dark' {
5− span.DecNumber { color: #AE81FF; }
6− span.BinNumber { color: #AE81FF; }
7− span.HexNumber { color: #AE81FF; }
8− span.OctNumber { color: #AE81FF; }
9− span.FloatNumber { color: #AE81FF; }
10− span.Identifier { color: #F8F8F2; }
11− span.Keyword { font-weight: 600; color: #F92672; }
12− span.StringLit { color: #E6DB74; }
13− span.LongStringLit { color: #E6DB74; }
14− span.CharLit { color: #E6DB74; }
15− span.EscapeSequence { color: white; }
16− span.Operator { color: white; }
17− span.Punctuation {color: white; }
18− span.Comment, span.LongComment {
19− font-style: italic;
20− font-weight: 400;
21− color: #75715E; }
22−
23− span.RegularExpression { color: darkviolet; }
24− span.TagStart { color: #F92672; }
25− span.TagEnd { color: #F92672; }
26− span.Key { color: #AE81FF; }
27− span.Value { color: #AE81FF; }
28− span.RawData { color: #a4255b; }
29− span.Assembler { color: #AE81FF; }
30− span.Preprocessor { color: #AE81FF; }
31− span.Directive { color: #AE81FF; }
32−
33− span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
34− span.Other { color: white; }
35−
36− /* Pop type, const, proc, and iterator defs in nim def blocks */
37− dt pre > span.Identifier, dt pre > span.Operator { color: #529B2F; font-weight: 700; }
38− } @else {
39− span.DecNumber { color: #252dbe; }
40− span.BinNumber { color: #252dbe; }
41− span.HexNumber { color: #252dbe; }
42− span.OctNumber { color: #252dbe; }
43− span.FloatNumber { color: #252dbe; }
44− span.Identifier { color: #3b3b3b; }
45− span.Keyword { font-weight: 600; color: #5e8f60; }
46− span.StringLit { color: #a4255b; }
47− span.LongStringLit { color: #a4255b; }
48− span.CharLit { color: #a4255b; }
49− span.EscapeSequence { color: black; }
50− span.Operator { color: black; }
51− span.Punctuation {color: black; }
52− span.Comment, span.LongComment {
53− font-style: italic;
54− font-weight: 400;
55− color: #484a86; }
56−
57− span.RegularExpression { color: darkviolet; }
58− span.TagStart { color: darkviolet; }
59− span.TagEnd { color: darkviolet; }
60− span.Key { color: #252dbe; }
61− span.Value { color: #252dbe; }
62− span.RawData { color: #a4255b; }
63− span.Assembler { color: #252dbe; }
64− span.Preprocessor { color: #252dbe; }
65− span.Directive { color: #252dbe; }
66−
67− span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
68− span.Other { color: black; }
69−
70− /* Pop type, const, proc, and iterator defs in nim def blocks */
71− dt pre > span.Identifier, dt pre > span.Operator { color: #155da4; font-weight: 700; }
72− }
73− dt pre > span.Identifier ~ span.Identifier, dt pre > span.Operator ~ span.Identifier {
74− color: inherit;
75− font-weight: inherit; }
76−
77− dt pre > span.Operator ~ span.Identifier, dt pre > span.Operator ~ span.Operator {
78− color: inherit;
79− font-weight: inherit; }
80−
81−}
assets/stylesheets/pages/_simple.scss+1−0
@@ -38,6 +38,7 @@ · 37 unmodified lines
3838 ._markdown,
3939 ._mocha,
4040 ._mongoose,
41+._nim,
4142 ._redux,
4243 ._requirejs,
4344 ._typescript,
lib/docs/filters/nim/clean_html.rb+46−12
@@ -4,26 +4,60 @@ module Docs · 3 unmodified lines
44 def call
55 @doc = at_css('#documentId .container')
66
7− css('.docinfo').remove
7+ css('.docinfo', '.footer', 'blockquote > p:empty', '.link-seesrc').remove
88
9− content = at_css('#content')
10− if content != nil
11− at_css('#content').remove_attribute('class')
12− @doc.add_child(at_css('#content').inner_html)
9+ css('h1:not(.title), h2, h3, h4').each do |node|
10+ node.name = node.name.sub(/\d/) { |i| i.to_i + 1 }
1311 end
1412
15− css('> div.row').remove
13+ if content = at_css('#content')
14+ content.prepend_child at_css('h1.title')
15+ @doc = content
16+ end
1617
17− css('pre').each do |node|
18− node['data-language'] = 'nim'
18+ if root_page?
19+ at_css('h1').content = 'Nim Documentation'
1920 end
2021
21− # remove link from headers
2222 css('h1 > a', 'h2 > a', 'h3 > a', 'h4 > a').each do |node|
23− node.parent['id'] = node['id']
24− node.parent.content = node.content
23+ node.parent['id'] = node['id'] if node['id']
24+ node.before(node.children).remove
25+ end
26+
27+ css('a[name]').each do |node|
28+ node.next_element['id'] = node['name']
29+ node.remove
30+ end
31+
32+ css('pre').each do |node|
33+ node.content = node.content.strip
34+ node['data-language'] = 'nim' unless node.content =~ /\A[\w\-\_\:\=\ ]+\z/
2535 end
26−
36+
37+ css('tt').each do |node|
38+ node.name = 'code'
39+ end
40+
41+ css('cite').each do |node|
42+ node.name = 'em'
43+ end
44+
45+ css('.section').each do |node|
46+ node.first_element_child['id'] = node['id'] if node['id']
47+ node.before(node.children).remove
48+ end
49+
50+ css('span.pre').each do |node|
51+ node.before(node.children).remove
52+ end
53+
54+ css('blockquote > pre:only-child', 'blockquote > dl:only-child', 'blockquote > table').each do |node|
55+ node.parent.before(node.parent.children).remove
56+ end
57+
58+ css('a', 'dl', 'table', 'code').remove_attr('class')
59+ css('table').remove_attr('border')
60+
2761 doc
2862 end
2963 end
lib/docs/filters/nim/entries.rb+44−31
@@ -1,49 +1,62 @@
11 module Docs
22 class Nim
33 class EntriesFilter < Docs::EntriesFilter
4− def get_type
5− at_css('h1').content
4+ def get_name
5+ name = at_css('h1').content
6+ name.remove! 'Module '
7+ name.remove! ' User Guide'
8+ name.remove! ' User\'s manual'
9+ name.remove! %r{ \-.*}
10+ name.strip!
11+ name
612 end
713
8− def get_name
9− at_css('h1').content
14+ def get_type
15+ if name.include?('Tutorial')
16+ 'Tutorial'
17+ elsif slug == 'manual'
18+ 'Manual'
19+ elsif at_css('h1').content.include?('Module ')
20+ name
21+ else
22+ 'Reference'
23+ end
1024 end
1125
1226 def additional_entries
1327 entries = []
14− if get_name.start_with? 'Module '
15− module_name = get_name[7..-1]
16− css('div .section').map do |node|
17− section_node = node.at_css('h1 a')
18− if section_node != nil
19− section_name = section_node.content.strip
20− items_node = node.at_css('dl.item')
21− if items_node != nil
22− items_node.css('dt a').map do |item_node|
23− item_name = item_node['name']
24− if item_name.include? ','
25− item_name = item_name.sub(',', '(') + ')'
26− end
27− entries << [module_name + '.' + item_name, item_node.parent['id']]
28− end
29− end
28+
29+ if at_css('h1').content.include?('Module ')
30+ css('#toc-list > li > .simple-toc-section').each do |node|
31+ type = node.previous_element.content.strip
32+
33+ node.css('a.reference:not(.reference-toplevel)').each do |n|
34+ n.css('span').remove
35+ name = n.content.strip
36+ name << '()' if (type == 'Procs' || type == 'Templates') && !name.include?('`')
37+ name.remove! '`'
38+ name.prepend "#{self.name}."
39+ id = n['href'].remove('#')
40+ entries << [name, id] unless entries.any? { |e| e[0] == name }
3041 end
3142 end
32− else
33− css('h1', 'h2', 'h3').map do |node|
34− id = node['id']
43+ elsif slug == 'manual'
44+ css('#toc-list > li > a').each do |node|
3545 name = node.content.strip
36− if id != nil
37− entries << [name, id]
38− else
39− a = node.at_css('a')
40− if a != nil
41− id = a['id']
42− entries << [name, id]
43− end
46+ next if name.start_with?('About')
47+ id = node['href'].remove('#')
48+ entries << [name, id]
49+ end
50+
51+ css('#toc-list > ul').each do |node|
52+ type = node.previous_element.content.strip
53+
54+ node.css('> li > a').each do |n|
55+ entries << [n.content.strip, n['href'].remove('#'), "Manual: #{type}"]
4456 end
4557 end
4658 end
59+
4760 entries
4861 end
4962 end
lib/docs/scrapers/nim.rb+8−8
@@ -2,20 +2,20 @@ module Docs · 1 unmodified line
22 class Nim < UrlScraper
33 self.type = 'nim'
44 self.release = '0.17.0'
5+ self.base_url = 'https://nim-lang.org/docs/'
6+ self.root_path = 'overview.html'
57 self.links = {
68 home: 'https://nim-lang.org/',
79 code: 'https://github.com/nim-lang/Nim'
810 }
9− self.base_url = 'https://nim-lang.org/'
10− self.root_path = 'docs/overview.html'
1111
12− html_filters.push 'nim/entries', 'nim/clean_html'
13−
14− options[:skip] = %w(cdn-cgi/l/email-protection docs/theindex.html docs/docgen.txt)
12+ html_filters.push 'nim/entries', 'nim/clean_html'
13+
14+ options[:skip] = %w(theindex.html docgen.txt)
15+
1516 options[:attribution] = <<-HTML
1617 &copy; 2006&ndash;2017 Andreas Rumpf<br>
17− All rights reserved. Licensed under the MIT License.
18+ Licensed under the MIT License.
1819 HTML
19−
2020 end
21−end
21+end
public/icons/docs/nim/16.png+0−0

Binary file not shown.

public/icons/docs/nim/16@2x.png+0−0

Binary file not shown.

public/icons/docs/nim/SOURCE+1−1
@@ -1 +1 @@
1−https://nim-lang.org/assets/img/logo.svg
1+https://github.com/nim-lang/website/tree/master/jekyll/assets/img