History

Finish D scraper

Thibaut Courouble committed 9 years ago

Commit 8902d531 parent bc8d94313 files · +192 −57Browse files

Unified · Split

13 files
  1. assets/images/docs-2.png +0 −0
  2. assets/images/docs-2@2x.png +0 −0
  3. assets/javascripts/news.json +1 −1
  4. assets/javascripts/templates/pages/about_tmpl.coffee +5 −0
  5. assets/javascripts/vendor/prism.js +65 −1
  6. assets/stylesheets/global/_icons.scss +1 −0
  7. assets/stylesheets/pages/_d.scss +5 −40
  8. lib/docs/filters/d/clean_html.rb +68 −2
  9. lib/docs/filters/d/entries.rb +32 −11
  10. lib/docs/scrapers/d.rb +18 −6
  11. public/icons/docs/d/16.png +0 −0
  12. public/icons/docs/d/16@2x.png +0 −0
  13. public/icons/docs/d/SOURCE +1 −0
assets/images/docs-2.png+0−0

Binary file not shown.

assets/images/docs-2@2x.png+0−0

Binary file not shown.

assets/javascripts/news.json+1−1
@@ -1,7 +1,7 @@
1[1[
2 [2 [
3 "2017-09-03",3 "2017-09-03",
4 "New documentations: <a href=\"/nim/\">Nim</a> and <a href=\"/vulkan/\">Vulkan</a>"4 "New documentations: <a href=\"/d/\">D</a>, <a href=\"/nim/\">Nim</a> and <a href=\"/vulkan/\">Vulkan</a>"
5 ], [5 ], [
6 "2017-07-23",6 "2017-07-23",
7 "New documentation: <a href=\"/godot/\">Godot</a>"7 "New documentation: <a href=\"/godot/\">Godot</a>"
assets/javascripts/templates/pages/about_tmpl.coffee+5−0
@@ -193,6 +193,11 @@ credits = [ · 192 unmodified lines
193 '2012-2017 Manas Technology Solutions',193 '2012-2017 Manas Technology Solutions',
194 'Apache',194 'Apache',
195 'https://raw.githubusercontent.com/crystal-lang/crystal/master/LICENSE'195 'https://raw.githubusercontent.com/crystal-lang/crystal/master/LICENSE'
196 ], [
197 'D',
198 '1999-2017 The D Language Foundation',
199 'Boost',
200 'https://raw.githubusercontent.com/dlang/phobos/master/LICENSE_1_0.txt'
196 ], [201 ], [
197 'D3.js',202 'D3.js',
198 '2010-2017 Michael Bostock',203 '2010-2017 Michael Bostock',
assets/javascripts/vendor/prism.js+65−1
@@ -1,4 +1,4 @@
1/* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+c+cpp+coffeescript+ruby+elixir+erlang+go+java+json+kotlin+lua+nginx+nim+perl+php+python+crystal+rust+scss+sql+typescript */1/* http://prismjs.com/download.html?themes=prism&languages=markup+css+clike+javascript+c+cpp+coffeescript+ruby+d+elixir+erlang+go+java+json+kotlin+lua+nginx+nim+perl+php+python+crystal+rust+scss+sql+typescript */
2var _self = (typeof window !== 'undefined')2var _self = (typeof window !== 'undefined')
3 ? window // if in browser3 ? window // if in browser
4 : (4 : (
@@ -976,6 +976,70 @@ delete Prism.languages.coffeescript['template-string']; · 971 unmodified lines
976 }976 }
977 ];977 ];
978}(Prism));978}(Prism));
979Prism.languages.d = Prism.languages.extend('clike', {
980 'string': [
981 // r"", x""
982 /\b[rx]"(\\.|[^\\"])*"[cwd]?/,
983 // q"[]", q"()", q"<>", q"{}"
984 /\bq"(?:\[[\s\S]*?\]|\([\s\S]*?\)|<[\s\S]*?>|\{[\s\S]*?\})"/,
985 // q"IDENT
986 // ...
987 // IDENT"
988 /\bq"([_a-zA-Z][_a-zA-Z\d]*)(?:\r?\n|\r)[\s\S]*?(?:\r?\n|\r)\1"/,
989 // q"//", q"||", etc.
990 /\bq"(.)[\s\S]*?\1"/,
991 // Characters
992 /'(?:\\'|\\?[^']+)'/,
993
994 /(["`])(\\.|(?!\1)[^\\])*\1[cwd]?/
995 ],
996
997 'number': [
998 // The lookbehind and the negative look-ahead try to prevent bad highlighting of the .. operator
999 // Hexadecimal numbers must be handled separately to avoid problems with exponent "e"
1000 /\b0x\.?[a-f\d_]+(?:(?!\.\.)\.[a-f\d_]*)?(?:p[+-]?[a-f\d_]+)?[ulfi]*/i,
1001 {
1002 pattern: /((?:\.\.)?)(?:\b0b\.?|\b|\.)\d[\d_]*(?:(?!\.\.)\.[\d_]*)?(?:e[+-]?\d[\d_]*)?[ulfi]*/i,
1003 lookbehind: true
1004 }
1005 ],
1006
1007 // In order: $, keywords and special tokens, globally defined symbols
1008 'keyword': /\$|\b(?:abstract|alias|align|asm|assert|auto|body|bool|break|byte|case|cast|catch|cdouble|cent|cfloat|char|class|const|continue|creal|dchar|debug|default|delegate|delete|deprecated|do|double|else|enum|export|extern|false|final|finally|float|for|foreach|foreach_reverse|function|goto|idouble|if|ifloat|immutable|import|inout|int|interface|invariant|ireal|lazy|long|macro|mixin|module|new|nothrow|null|out|override|package|pragma|private|protected|public|pure|real|ref|return|scope|shared|short|static|struct|super|switch|synchronized|template|this|throw|true|try|typedef|typeid|typeof|ubyte|ucent|uint|ulong|union|unittest|ushort|version|void|volatile|wchar|while|with|__(?:(?:FILE|MODULE|LINE|FUNCTION|PRETTY_FUNCTION|DATE|EOF|TIME|TIMESTAMP|VENDOR|VERSION)__|gshared|traits|vector|parameters)|string|wstring|dstring|size_t|ptrdiff_t)\b/,
1009 'operator': /\|[|=]?|&[&=]?|\+[+=]?|-[-=]?|\.?\.\.|=[>=]?|!(?:i[ns]\b|<>?=?|>=?|=)?|\bi[ns]\b|(?:<[<>]?|>>?>?|\^\^|[*\/%^~])=?/
1010});
1011
1012
1013Prism.languages.d.comment = [
1014 // Shebang
1015 /^\s*#!.+/,
1016 // /+ +/
1017 {
1018 // Allow one level of nesting
1019 pattern: /(^|[^\\])\/\+(?:\/\+[\s\S]*?\+\/|[\s\S])*?\+\//,
1020 lookbehind: true
1021 }
1022].concat(Prism.languages.d.comment);
1023
1024Prism.languages.insertBefore('d', 'comment', {
1025 'token-string': {
1026 // Allow one level of nesting
1027 pattern: /\bq\{(?:|\{[^}]*\}|[^}])*\}/,
1028 alias: 'string'
1029 }
1030});
1031
1032Prism.languages.insertBefore('d', 'keyword', {
1033 'property': /\B@\w*/
1034});
1035
1036Prism.languages.insertBefore('d', 'function', {
1037 'register': {
1038 // Iasm registers
1039 pattern: /\b(?:[ABCD][LHX]|E[ABCD]X|E?(?:BP|SP|DI|SI)|[ECSDGF]S|CR[0234]|DR[012367]|TR[3-7]|X?MM[0-7]|R[ABCD]X|[BS]PL|R[BS]P|[DS]IL|R[DS]I|R(?:[89]|1[0-5])[BWD]?|XMM(?:[89]|1[0-5])|YMM(?:1[0-5]|\d))\b|\bST(?:\([0-7]\)|\b)/,
1040 alias: 'variable'
1041 }
1042});
979Prism.languages.elixir = {1043Prism.languages.elixir = {
980 // Negative look-ahead is needed for string interpolation1044 // Negative look-ahead is needed for string interpolation
981 // Negative look-behind is needed to avoid highlighting markdown headers in1045 // Negative look-behind is needed to avoid highlighting markdown headers in
assets/stylesheets/global/_icons.scss+1−0
@@ -175,3 +175,4 @@ · 174 unmodified lines
175._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }175._icon-godot:before { background-position: -4rem -2rem; @extend %doc-icon-2; }
176._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }176._icon-nim:before { background-position: -5rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
177._icon-vulkan:before { background-position: -6rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }177._icon-vulkan:before { background-position: -6rem -2rem; @extend %doc-icon-2; @extend %darkIconFix !optional; }
178._icon-d:before { background-position: -7rem -2rem; @extend %doc-icon-2; }
assets/stylesheets/pages/_d.scss+5−40
@@ -1,44 +1,9 @@
1._d {1._d {
2 > .description, > .documentation-section { padding-left: 1rem; }2 h2 { @extend %block-heading; }
3 > .description > h2, header > h3, > h2 { @extend %block-heading; }3 h3, .d_decl { @extend %block-label, %label-blue; }
4 .description > h1 { font-size: 1rem; }4 .d_decl { @extend %code; }
5 .method-description > h2, h3, h4, h5, h6 { font-size: 1em; }
6 5
7 .d_decl {6 p > code, li > code, td > code, dd > code { @extend %label; }
8 font-weight: $boldFontWeight;
9 @extend %block-label, %label-blue;
10 7
11 + .d_decl { margin-top: -.5em; }8 span.red { color: $textColorRed; }
12 }
13
14 > .meta {
15 @extend %note, %note-blue;
16
17 > dd { margin: 0; }
18 > dd + dt { margin-top: .5em; }
19 }
20
21 a.method-click-advice {
22 float: right;
23 font-size: .75rem;
24 color: $linkColor;
25 cursor: pointer;
26 @extend %user-select-none;
27
28 &:hover { text-decoration: underline; }
29 }
30
31 .method-description { position: relative; }
32
33 .method-source-code {
34 display: none;
35 position: absolute;
36 z-index: 1;
37 top: 0;
38 right: 0;
39 background: rgba($contentBackground, .95);
40 box-shadow: 0 1em 1em 1em $contentBackground;
41
42 > pre { margin: 0; }
43 }
44}9}
lib/docs/filters/d/clean_html.rb+68−2
@@ -2,9 +2,75 @@ module Docs · 1 unmodified line
2 class D2 class D
3 class CleanHtmlFilter < Filter3 class CleanHtmlFilter < Filter
4 def call4 def call
5 css('.d_decl > div > span.def-anchor').each do |node|5 @doc = at_css("#content")
6 node.parent.parent['id'] = node['id']6
7 css('#tools', '#copyright').remove
8
9 css('td > b', 'h1 > span').each do |node|
10 node.before(node.children).remove
7 end11 end
12
13 css('span.d_inlinecode').each do |node|
14 node.name = 'code'
15 node.remove_attribute('class')
16 end
17
18 css('.keyval').each do |node|
19 key = node.at_css('.key')
20 dt = key.inner_html
21 dd = if val = node.at_css('.val')
22 val.inner_html
23 else
24 siblings = []
25 siblings << key while key = key.next
26 siblings.map(&:to_html).join
27 end
28 node.replace("<dl><dt>#{dt}</dt><dd>#{dd}</dd></dl>")
29 end
30
31 css('div.summary', 'div.description').each do |node|
32 node.name = 'p' unless node.at_css('p')
33 node.css('.blankline').each { |n| n.replace('<br><br>') }
34 end
35
36 css('.d_decl').each do |node|
37 node['id'] = node.at_css('.def-anchor')['id'].remove(/\A\./)
38 constraints = node.css('.constraint').remove
39 node.content = node.content.strip
40 node.inner_html = node.inner_html.gsub(/;\s*/, '<br>').remove(/<br>\z/)
41 node << "<br><br> Constraints:<br> #{constraints.map(&:content).join('<br> ')}" unless constraints.empty?
42 end
43
44 css('pre').each do |node|
45 node.content = node.content
46 node['data-language'] = 'd' if node['class'] && node['class'].include?('d_code')
47 end
48
49 css('div', 'code > a > code', 'code > code').each do |node|
50 node.before(node.children).remove
51 end
52
53 css('a[href*="#."]').each do |node|
54 node['href'] = node['href'].sub('#.', '#')
55 end
56
57 css('tr', 'td', 'code', 'pre', 'p', 'table').remove_attr('class')
58 css('table').remove_attr('border').remove_attr('cellpadding').remove_attr('cellspacing')
59
60 if base_url.path == '/spec/'
61 css('a.anchor').each do |node|
62 node.parent['id'] ||= node['id']
63 node.before(node.children).remove
64 end
65
66 css('center').each do |node|
67 node.before(node.children).remove
68 end
69
70 css('.fa-angle-left + a').remove
71 css('a + .fa-angle-right').each { |node| node.previous_element.remove }
72 end
73
8 doc74 doc
9 end75 end
10 end76 end
lib/docs/filters/d/entries.rb+32−11
@@ -2,25 +2,46 @@ module Docs · 1 unmodified line
2 class D2 class D
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 slug.to_s.gsub('_', '.').gsub('/', '.').squish!5 name = at_css('h1').content
6
7 if base_url.path == '/spec/'
8 index = css('.subnav li a').to_a.index(at_css(".subnav li a[href='#{result[:path]}']")) + 1
9 name.prepend "#{index}. "
10 end
11
12 name
6 end13 end
7 14
8 def get_type15 def get_type
9 slug.to_s.sub(/_(.*)/, '')16 return 'Reference' if base_url.path == '/spec/'
17
18 if name.start_with?('etc') || name.start_with?('core.stdc.')
19 name.split('.')[0..2].join('.')
20 elsif name.start_with?('ddmd')
21 'ddmd'
22 else
23 name.split('.')[0..1].join('.')
24 end
10 end25 end
11 26
12 def additional_entries27 def additional_entries
13 names = []28 return [] if root_page? || base_url.path == '/spec/'
14 css('.book > tr > td > a').each do |x|29
15 span_block = x.at_css('span')30 entries = []
16 if span_block != nil31
17 elem_name = span_block.text32 css('.book > tr > td > a').each do |node|
18 name = "#{get_name}.#{elem_name}"33 entries << ["#{self.name}.#{node.content}", node['href'].remove(/\A#/).remove(/\A\./)]
19 type = name.sub(/\..*/,'')34 end
20 names << [name, "#{slug}#{x['href']}", type]35
36 if entries.empty?
37 css('.quickindex[id]').each do |node|
38 name = node['id'].remove(/quickindex\.?/)
39 next if name.empty? || name =~ /\.\d+\z/
40 entries << ["#{self.name}.#{name}", name]
21 end41 end
22 end42 end
23 names43
44 entries
24 end45 end
25 end46 end
26 end47 end
lib/docs/scrapers/d.rb+18−6
@@ -1,18 +1,30 @@
1module Docs1module Docs
2 class D < UrlScraper2 class D < UrlScraper
3 self.release = '2.075.1'3 include MultipleBaseUrls
4
5 self.release = '2.076.0'
4 self.type = 'd'6 self.type = 'd'
5 self.base_url = 'http://dlang.org/phobos/'7 self.base_urls = ['https://dlang.org/phobos/', 'https://dlang.org/spec/']
8 self.root_path = 'index.html'
9 self.links = {
10 home: 'https://dlang.org/',
11 code: 'https://github.com/dlang/phobos'
12 }
6 13
7 html_filters.push 'd/entries', 'd/clean_html'14 html_filters.push 'd/entries', 'd/clean_html'
8 15
9 options[:container] = '#content'16 options[:skip] = %w(spec.html)
17 options[:container] = '.container'
18 options[:root_title] = 'D Programming Language'
10 options[:title] = false19 options[:title] = false
11 options[:root_title] = 'D Language'
12 options[:skip_patterns] = [/#.*/]
13 20
14 options[:attribution] = <<-HTML21 options[:attribution] = <<-HTML
15 Copyright &copy; 1999-2017 by the D Language Foundation22 &copy; 1999&ndash;2017 The D Language Foundation<br>
23 Licensed under the Boost License 1.0.
16 HTML24 HTML
25
26 def initial_urls
27 %w(https://dlang.org/phobos/index.html https://dlang.org/spec/intro.html)
28 end
17 end29 end
18end30end
public/icons/docs/d/16.png+0−0

Binary file not shown.

public/icons/docs/d/16@2x.png+0−0

Binary file not shown.

public/icons/docs/d/SOURCE+1−0
@@ -0,0 +1 @@
1https://github.com/dlang/dlang.org/tree/master/images