History

Improve Erlang scraper

Closes #432.

Thibaut Courouble committed 10 years ago

Commit 23463001 parent 368eda35 files · +71 −20Browse files

Unified · Split

5 files
  1. assets/stylesheets/pages/_erlang.scss +1 −0
  2. lib/docs/filters/erlang/clean_html.rb +3 −7
  3. lib/docs/filters/erlang/entries.rb +50 −11
  4. lib/docs/filters/erlang/pre_clean_html.rb +11 −1
  5. lib/docs/scrapers/erlang.rb +6 −1
assets/stylesheets/pages/_erlang.scss+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
2 @extend %simple;2 @extend %simple;
3 3
4 h3.code { @extend %code; }4 h3.code { @extend %code; }
5 code.code { @extend %label; }
5 .note { @extend %note; }6 .note { @extend %note; }
6 .warning { @extend %note, %note-red; }7 .warning { @extend %note, %note-red; }
7 .note .label, .warning .label { font-weight: bold; }8 .note .label, .warning .label { font-weight: bold; }
lib/docs/filters/erlang/clean_html.rb+3−7
@@ -45,7 +45,7 @@ module Docs · 44 unmodified lines
45 node.content = content.capitalize if content == content.upcase45 node.content = content.capitalize if content == content.upcase
46 end46 end
47 47
48 css('p > span.bold_code:first-child ~ br:last-child').each do |node|48 css('p > .bold_code:first-child ~ br:last-child').each do |node|
49 parent = node.parent49 parent = node.parent
50 parent.name = 'h3'50 parent.name = 'h3'
51 parent['class'] = 'code'51 parent['class'] = 'code'
@@ -54,15 +54,11 @@ module Docs · 2 unmodified lines
54 parent.inner_html = parent.inner_html.strip54 parent.inner_html = parent.inner_html.strip
55 end55 end
56 56
57 css('span.code').each do |node|57 css('pre:not(.REFTYPES) *:not(a)', 'a[href^=javascript]').each do |node|
58 node.name = 'code'
59 end
60
61 css('pre *:not(a)').each do |node|
62 node.before(node.children).remove58 node.before(node.children).remove
63 end59 end
64 60
65 css('pre').each do |node|61 css('pre:not(.REFTYPES)').each do |node|
66 node.inner_html = node.inner_html.strip_heredoc62 node.inner_html = node.inner_html.strip_heredoc
67 end63 end
68 64
lib/docs/filters/erlang/entries.rb+50−11
@@ -3,14 +3,32 @@ module Docs · 2 unmodified lines
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 def get_name4 def get_name
5 name = at_css('h1').content.strip5 name = at_css('h1').content.strip
6 name.prepend 'Guide: ' if doc.inner_html.include?('<strong>User\'s Guide</strong>')6 name << " (#{type.remove('Guide: ')})" if name == '1 Introduction'
7 name7 name
8 end8 end
9 9
10 def get_type10 def get_type
11 type = subpath[/lib\/(.+?)[\-\/]/, 1]11 if subpath.start_with?('lib/')
12 type << "/#{name}" if type == 'stdlib' && entry_nodes.length >= 1012 type = subpath[/lib\/(.+?)[\-\/]/, 1]
13 type13 type << "/#{name}" if type == 'stdlib' && entry_nodes.length >= 10
14 type
15 elsif subpath.start_with?('doc/')
16 type = subpath[/doc\/(.+?)\//, 1]
17 type.capitalize!
18 type.sub! '_', ' '
19 type.sub! 'Oam', 'OAM'
20 type.remove! ' Guide'
21 type.prepend 'Guide: '
22 type
23 elsif subpath.start_with?('erts')
24 type = 'ERTS'
25 if name =~ /\A\d/
26 type.prepend 'Guide: '
27 elsif entry_nodes.length > 0
28 type << "/#{name}"
29 end
30 type
31 end
14 end32 end
15 33
16 def include_default_entry?34 def include_default_entry?
@@ -18,17 +36,38 @@ module Docs · 1 unmodified line
18 end36 end
19 37
20 def additional_entries38 def additional_entries
21 entry_nodes.map do |node|39 return [] unless include_default_entry?
22 id = node['name']40
23 name = id.gsub %r{\-(?<arity>.*)\z}, '/\k<arity>'41 if subpath.start_with?('lib/')
24 name.remove! 'Module:'42 entry_nodes.map do |node|
25 name.prepend "#{self.name}:"43 id = node['name']
26 [name, id]44 name = id.gsub %r{\-(?<arity>.*)\z}, '/\k<arity>'
45 name.remove! 'Module:'
46 name.prepend "#{self.name}:"
47 [name, id]
48 end
49 elsif subpath.start_with?('doc/')
50 []
51 elsif subpath.start_with?('erts')
52 return [] if type.start_with?('Guide')
53 entry_nodes.map do |node|
54 id = node['href'][/#(.+)/, 1]
55 name = node.content.strip
56 name.remove! 'Module:'
57 name.prepend "#{self.name}:"
58 [name, id]
59 end
27 end60 end
28 end61 end
29 62
30 def entry_nodes63 def entry_nodes
31 @entry_nodes ||= css('div.REFBODY + p > a')64 @entry_nodes ||= if subpath.start_with?('lib/')
65 css('div.REFBODY + p > a')
66 elsif subpath.start_with?('erts')
67 link = at_css(".flipMenu a[href='#{File.basename(subpath, '.html')}']")
68 list = link.parent.parent
69 list['class'] == 'flipMenu' ? [] : list.css('a').to_a.tap { |a| a.delete(link); }
70 end
32 end71 end
33 end72 end
34 end73 end
lib/docs/filters/erlang/pre_clean_html.rb+11−1
@@ -2,7 +2,17 @@ module Docs · 1 unmodified line
2 class Erlang2 class Erlang
3 class PreCleanHtmlFilter < Filter3 class PreCleanHtmlFilter < Filter
4 def call4 def call
5 css('.flipMenu li[title] > a').remove5 css('.flipMenu li[title] > a').remove unless subpath.start_with?('erts') # perf
6
7 css('.REFTYPES').each do |node|
8 node.name = 'pre'
9 end
10
11 css('span.bold_code', 'span.code').each do |node|
12 node.name = 'code'
13 node.inner_html = node.inner_html.strip.gsub(/\s+/, ' ')
14 end
15
6 doc16 doc
7 end17 end
8 end18 end
lib/docs/scrapers/erlang.rb+6−1
@@ -10,13 +10,18 @@ module Docs · 9 unmodified lines
10 html_filters.insert_after 'container', 'erlang/pre_clean_html'10 html_filters.insert_after 'container', 'erlang/pre_clean_html'
11 html_filters.push 'erlang/entries', 'erlang/clean_html'11 html_filters.push 'erlang/entries', 'erlang/clean_html'
12 12
13 options[:only_patterns] = [/\Alib/]13 options[:only_patterns] = [
14 /\Alib/,
15 /\Adoc\/\w+\//,
16 /\Aerts.+\/html/
17 ]
14 18
15 options[:skip_patterns] = [19 options[:skip_patterns] = [
16 /pdf/,20 /pdf/,
17 /release_notes/,21 /release_notes/,
18 /result/,22 /result/,
19 /java/,23 /java/,
24 /\.erl\z/,
20 /\/html\/.*_app\.html\z/,25 /\/html\/.*_app\.html\z/,
21 /_examples\.html\z/,26 /_examples\.html\z/,
22 /\Alib\/edoc/,27 /\Alib\/edoc/,