History

Improve Erlang scraper

Closes #432.

Thibaut Courouble committed 10 years ago

Commit 23463001 parent 368eda35 files · +71 −20Browse files

Unified · Split

5 files
  1. assets/stylesheets/pages/_erlang.scss +1 −0
  2. lib/docs/filters/erlang/clean_html.rb +3 −7
  3. lib/docs/filters/erlang/entries.rb +50 −11
  4. lib/docs/filters/erlang/pre_clean_html.rb +11 −1
  5. lib/docs/scrapers/erlang.rb +6 −1
assets/stylesheets/pages/_erlang.scss+1−0
@@ -2,6 +2,7 @@ · 1 unmodified line
22 @extend %simple;
33
44 h3.code { @extend %code; }
5+ code.code { @extend %label; }
56 .note { @extend %note; }
67 .warning { @extend %note, %note-red; }
78 .note .label, .warning .label { font-weight: bold; }
lib/docs/filters/erlang/clean_html.rb+3−7
@@ -45,7 +45,7 @@ module Docs · 44 unmodified lines
4545 node.content = content.capitalize if content == content.upcase
4646 end
4747
48− css('p > span.bold_code:first-child ~ br:last-child').each do |node|
48+ css('p > .bold_code:first-child ~ br:last-child').each do |node|
4949 parent = node.parent
5050 parent.name = 'h3'
5151 parent['class'] = 'code'
@@ -54,15 +54,11 @@ module Docs · 2 unmodified lines
5454 parent.inner_html = parent.inner_html.strip
5555 end
5656
57− css('span.code').each do |node|
58− node.name = 'code'
59− end
60−
61− css('pre *:not(a)').each do |node|
57+ css('pre:not(.REFTYPES) *:not(a)', 'a[href^=javascript]').each do |node|
6258 node.before(node.children).remove
6359 end
6460
65− css('pre').each do |node|
61+ css('pre:not(.REFTYPES)').each do |node|
6662 node.inner_html = node.inner_html.strip_heredoc
6763 end
6864
lib/docs/filters/erlang/entries.rb+50−11
@@ -3,14 +3,32 @@ module Docs · 2 unmodified lines
33 class EntriesFilter < Docs::EntriesFilter
44 def get_name
55 name = at_css('h1').content.strip
6− name.prepend 'Guide: ' if doc.inner_html.include?('<strong>User\'s Guide</strong>')
6+ name << " (#{type.remove('Guide: ')})" if name == '1 Introduction'
77 name
88 end
99
1010 def get_type
11− type = subpath[/lib\/(.+?)[\-\/]/, 1]
12− type << "/#{name}" if type == 'stdlib' && entry_nodes.length >= 10
13− type
11+ if subpath.start_with?('lib/')
12+ type = subpath[/lib\/(.+?)[\-\/]/, 1]
13+ type << "/#{name}" if type == 'stdlib' && entry_nodes.length >= 10
14+ type
15+ elsif subpath.start_with?('doc/')
16+ type = subpath[/doc\/(.+?)\//, 1]
17+ type.capitalize!
18+ type.sub! '_', ' '
19+ type.sub! 'Oam', 'OAM'
20+ type.remove! ' Guide'
21+ type.prepend 'Guide: '
22+ type
23+ elsif subpath.start_with?('erts')
24+ type = 'ERTS'
25+ if name =~ /\A\d/
26+ type.prepend 'Guide: '
27+ elsif entry_nodes.length > 0
28+ type << "/#{name}"
29+ end
30+ type
31+ end
1432 end
1533
1634 def include_default_entry?
@@ -18,17 +36,38 @@ module Docs · 1 unmodified line
1836 end
1937
2038 def additional_entries
21− entry_nodes.map do |node|
22− id = node['name']
23− name = id.gsub %r{\-(?<arity>.*)\z}, '/\k<arity>'
24− name.remove! 'Module:'
25− name.prepend "#{self.name}:"
26− [name, id]
39+ return [] unless include_default_entry?
40+
41+ if subpath.start_with?('lib/')
42+ entry_nodes.map do |node|
43+ id = node['name']
44+ name = id.gsub %r{\-(?<arity>.*)\z}, '/\k<arity>'
45+ name.remove! 'Module:'
46+ name.prepend "#{self.name}:"
47+ [name, id]
48+ end
49+ elsif subpath.start_with?('doc/')
50+ []
51+ elsif subpath.start_with?('erts')
52+ return [] if type.start_with?('Guide')
53+ entry_nodes.map do |node|
54+ id = node['href'][/#(.+)/, 1]
55+ name = node.content.strip
56+ name.remove! 'Module:'
57+ name.prepend "#{self.name}:"
58+ [name, id]
59+ end
2760 end
2861 end
2962
3063 def entry_nodes
31− @entry_nodes ||= css('div.REFBODY + p > a')
64+ @entry_nodes ||= if subpath.start_with?('lib/')
65+ css('div.REFBODY + p > a')
66+ elsif subpath.start_with?('erts')
67+ link = at_css(".flipMenu a[href='#{File.basename(subpath, '.html')}']")
68+ list = link.parent.parent
69+ list['class'] == 'flipMenu' ? [] : list.css('a').to_a.tap { |a| a.delete(link); }
70+ end
3271 end
3372 end
3473 end
lib/docs/filters/erlang/pre_clean_html.rb+11−1
@@ -2,7 +2,17 @@ module Docs · 1 unmodified line
22 class Erlang
33 class PreCleanHtmlFilter < Filter
44 def call
5− css('.flipMenu li[title] > a').remove
5+ css('.flipMenu li[title] > a').remove unless subpath.start_with?('erts') # perf
6+
7+ css('.REFTYPES').each do |node|
8+ node.name = 'pre'
9+ end
10+
11+ css('span.bold_code', 'span.code').each do |node|
12+ node.name = 'code'
13+ node.inner_html = node.inner_html.strip.gsub(/\s+/, ' ')
14+ end
15+
616 doc
717 end
818 end
lib/docs/scrapers/erlang.rb+6−1
@@ -10,13 +10,18 @@ module Docs · 9 unmodified lines
1010 html_filters.insert_after 'container', 'erlang/pre_clean_html'
1111 html_filters.push 'erlang/entries', 'erlang/clean_html'
1212
13− options[:only_patterns] = [/\Alib/]
13+ options[:only_patterns] = [
14+ /\Alib/,
15+ /\Adoc\/\w+\//,
16+ /\Aerts.+\/html/
17+ ]
1418
1519 options[:skip_patterns] = [
1620 /pdf/,
1721 /release_notes/,
1822 /result/,
1923 /java/,
24+ /\.erl\z/,
2025 /\/html\/.*_app\.html\z/,
2126 /_examples\.html\z/,
2227 /\Alib\/edoc/,