History

migrate cpp scraper from filescraper to urlscraper

Enoc committed 5 years ago

Commit e17bc841 parent a754d509 files · +189 −171Browse files

Unified · Split

9 files
  1. lib/docs/filters/cpp/entries.rb +19 −1
  2. lib/docs/filters/cpp/fix_urls.rb +0 −11
  3. lib/docs/filters/cpp20/clean_html.rb +0 −9
  4. lib/docs/filters/cpp20/entries.rb +0 −82
  5. lib/docs/filters/cppref/clean_html.rb +116 −0
  6. lib/docs/filters/cppref/fix_code.rb +21 −0
  7. lib/docs/scrapers/cpp.rb +0 −52
  8. lib/docs/scrapers/cppref/cpp.rb +4 −16
  9. lib/docs/scrapers/cppref/cppref.rb +29 −0
lib/docs/filters/cpp/entries.rb+19−1
@@ -1,6 +1,8 @@
1module Docs1module Docs
2 class Cpp2 class Cpp
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4 @@duplicate_names = []
5
4 REPLACE_NAMES = {6 REPLACE_NAMES = {
5 'Error directive' => '#error directive',7 'Error directive' => '#error directive',
6 'Filename and line information' => '#line directive',8 'Filename and line information' => '#line directive',
@@ -11,7 +13,8 @@ module Docs · 4 unmodified lines
11 def get_name13 def get_name
12 name = at_css('#firstHeading').content.strip14 name = at_css('#firstHeading').content.strip
13 name = format_name(name)15 name = format_name(name)
14 name.split(',').first16 name = name.split(',').first
17 name
15 end18 end
16 19
17 def get_type20 def get_type
@@ -61,6 +64,21 @@ module Docs · 43 unmodified lines
61 64
62 REPLACE_NAMES[name] || name65 REPLACE_NAMES[name] || name
63 end66 end
67
68 # Avoid duplicate pages, these duplicate page are the same page for
69 # multiple functions that are organized in the same page because provide
70 # similar behavior but have different name.
71 def entries
72 entries = []
73
74 if !(@@duplicate_names.include?(name))
75 @@duplicate_names.push(name)
76 entries << default_entry if root_page? || include_default_entry?
77 entries.concat(additional_entries)
78 build_entries(entries)
79 end
80 end
81
64 end82 end
65 end83 end
66end84end
lib/docs/filters/cpp/fix_urls.rb+0−11
@@ -1,11 +0,0 @@
1module Docs
2 class Cpp
3 class FixUrlsFilter < Filter
4 def call
5 html.gsub! File.join(Cpp.base_url, Cpp.root_path), Cpp.base_url[0..-2]
6 html.gsub! %r{#{Cpp.base_url}([^"']+?)\.html}, "#{Cpp.base_url}\\1"
7 html
8 end
9 end
10 end
11end
lib/docs/filters/cpp20/clean_html.rb+0−9
@@ -1,9 +0,0 @@
1module Docs
2 class Cpp20
3 class CleanHtmlFilter < Filter
4 def call
5 doc
6 end
7 end
8 end
9end
lib/docs/filters/cpp20/entries.rb+0−82
@@ -1,82 +0,0 @@
1module Docs
2 class Cpp20
3 class EntriesFilter < Docs::EntriesFilter
4 @@duplicate_names = []
5
6 REPLACE_NAMES = {
7 'Error directive' => '#error directive',
8 'Filename and line information' => '#line directive',
9 'Implementation defined behavior control' => '#pragma directive',
10 'Replacing text macros' => '#define directive',
11 'Source file inclusion' => '#include directive' }
12
13 def get_name
14 name = at_css('#firstHeading').content.strip
15 name = format_name(name)
16 name = name.split(',').first
17 name
18 end
19
20 def get_type
21 if at_css('#firstHeading').content.include?('C++ keyword')
22 'Keywords'
23 elsif subpath.start_with?('experimental')
24 'Experimental libraries'
25 elsif subpath.start_with?('language/')
26 'Language'
27 elsif subpath.start_with?('freestanding')
28 'Utilities'
29 elsif type = at_css('.t-navbar > div:nth-child(4) > :first-child').try(:content)
30 type.strip!
31 type.remove! ' library'
32 type.remove! ' utilities'
33 type.remove! 'C++ '
34 type.capitalize!
35 type
36 end
37 end
38
39 def additional_entries
40 return [] if root_page? || self.name.start_with?('operators')
41 names = at_css('#firstHeading').content.remove(%r{\(.+?\)}).split(', ')[1..-1]
42 names.each(&:strip!).reject! do |name|
43 name.size <= 2 || name == '...' || name =~ /\A[<>]/ || name.start_with?('operator')
44 end
45 names.map { |name| [format_name(name)] }
46 end
47
48 def format_name(name)
49 name.remove! 'C++ concepts: '
50 name.remove! 'C++ keywords: '
51 name.remove! 'C++ ' unless name == 'C++ language'
52 name.remove! %r{\s\(.+\)}
53
54 name.sub! %r{\AStandard library header <(.+)>\z}, '\1'
55 name.sub! %r{(<[^>]+>)}, ''
56
57 if name.include?('operator') && name.include?(',')
58 name.sub!(%r{operator.+([\( ])}, 'operators (') || name.sub!(%r{operator.+}, 'operators')
59 name.sub! ' ', ' '
60 name << ')' unless name.last == ')' || name.exclude?('(')
61 name.sub! '()', ''
62 name.sub! %r{\(.+\)}, '' if !name.start_with?('operator') && name.length > 50
63 end
64
65 REPLACE_NAMES[name] || name
66 end
67
68 def entries
69 entries = []
70
71 # avoid duplicate pages
72 if !(@@duplicate_names.include?(name))
73 @@duplicate_names.push(name)
74 entries << default_entry if root_page? || include_default_entry?
75 entries.concat(additional_entries)
76 build_entries(entries)
77 end
78 end
79
80 end
81 end
82end
lib/docs/filters/cppref/clean_html.rb+116−0
@@ -0,0 +1,116 @@
1module Docs
2 class Cppref
3 class CleanHtmlFilter < Filter
4 def call
5 css('h1').remove if root_page?
6
7 css('.t-dcl-rev-aux td[rowspan]').each do |node|
8 rowspan = node['rowspan'].to_i
9 node['rowspan'] = node.ancestors('tbody').css('tr').length if rowspan > 3
10 end
11
12 css('#siteSub', '#contentSub', '.printfooter', '.t-navbar', '.editsection', '#toc',
13 '.t-dsc-sep', '.t-dcl-sep', '#catlinks', '.ambox-notice', '.mw-cite-backlink',
14 '.t-sdsc-sep:first-child:last-child', '.t-example-live-link',
15 '.t-dcl-rev-num > .t-dcl-rev-aux ~ tr:not(.t-dcl-rev-aux) > td:nth-child(2)').remove
16
17 css('#bodyContent', '.mw-content-ltr', 'span[style]', 'div[class^="t-ref"]', '.t-image',
18 'th > div', 'td > div', '.t-dsc-see', '.mainpagediv', 'code > b', 'tbody').each do |node|
19 node.before(node.children).remove
20 end
21
22 css('div > ul').each do |node|
23 node.parent.before(node.parent.children).remove
24 end
25
26 css('dl > dd:first-child:last-child > ul:first-child:last-child').each do |node|
27 dl = node.parent.parent
28 if dl.previous_element && dl.previous_element.name == 'ul'
29 dl.previous_element << node
30 dl.remove
31 else
32 dl.before(node).remove
33 end
34 end
35
36 css('dl > dd:first-child:last-child').each do |node|
37 node.parent.before(node.children).remove
38 end
39
40 css('ul').each do |node|
41 while node.next_element && node.next_element.name == 'ul'
42 node << node.next_element.children
43 node.next_element.remove
44 end
45 end
46
47 css('h2 > span[id]', 'h3 > span[id]', 'h4 > span[id]', 'h5 > span[id]', 'h6 > span[id]').each do |node|
48 node.parent['id'] = node['id']
49 node.before(node.children).remove
50 end
51
52 css('table[style]', 'th[style]', 'td[style]').remove_attr('style')
53 css('table[cellpadding]').remove_attr('cellpadding')
54
55 css('.t-dsc-hitem > td', '.t-dsc-header > td').each do |node|
56 node.name = 'th'
57 node.content = ' ' if node.content.empty?
58 end
59
60 css('tt', 'span > span.source-cpp', 'span.t-c', 'span.t-lc', 'span.t-dsc-see-tt').each do |node|
61 node.name = 'code'
62 node.remove_attribute('class')
63 node.content = node.content unless node.at_css('a')
64 end
65
66 css('div > span.source-cpp').each do |node|
67 node.name = 'pre'
68 node.inner_html = node.inner_html.gsub('<br>', "\n")
69 node.content = node.content
70 end
71
72 css('div > a > img[alt="About this image"]').each do |node|
73 node.parent.parent.remove
74 end
75
76 css('area[href]').each do |node|
77 node['href'] = node['href'].remove('.html')
78 end
79
80 css('p').each do |node|
81 while node.next && (node.next.text? || node.next.name == 'a' || node.next.name == 'code')
82 node << node.next
83 end
84 node.inner_html = node.inner_html.strip
85 node << '.' if node.content =~ /[a-zA-Z0-9\)]\z/
86 node.remove if node.content.blank? && !node.at_css('img')
87 end
88
89 css('pre').each do |node|
90 node['data-language'] = if node['class'].try(:include?, 'cpp') || node.parent['class'].try(:include?, 'cpp')
91 'cpp'
92 else
93 'c'
94 end
95 node.remove_attribute('class')
96 node.content = node.content.gsub("\t", ' ' * 8)
97 end
98
99 css('code code', '.mw-geshi').each do |node|
100 node.before(node.children).remove
101 end
102
103 css('h1 ~ .fmbox').each do |node|
104 node.name = 'div'
105 node.content = node.content
106 end
107
108 css('img').each do |node|
109 node['src'] = node['src'].sub! %r{http://en.cppreference.com/common/([^"']+?)\.svg}, 'http://upload.cppreference.com/mwiki/\1.svg'
110 end
111
112 doc
113 end
114 end
115 end
116end
lib/docs/filters/cppref/fix_code.rb+21−0
@@ -0,0 +1,21 @@
1module Docs
2 class Cppref
3 class FixCodeFilter < Filter
4 def call
5 css('div > span.source-c', 'div > span.source-cpp').each do |node|
6 node.inner_html = node.inner_html.gsub(/<br>\n?/, "\n").gsub("\n</p>\n", "</p>\n")
7 node.parent.name = 'pre'
8 node.parent['class'] = node['class']
9 node.parent.content = node.content
10 end
11
12 nbsp = Nokogiri::HTML('&nbsp;').text
13 css('pre').each do |node|
14 node.content = node.content.gsub(nbsp, ' ')
15 end
16
17 doc
18 end
19 end
20 end
21end
lib/docs/scrapers/cpp.rb+0−52
@@ -1,52 +0,0 @@
1module Docs
2 class Cpp < FileScraper
3 self.name = 'C++'
4 self.slug = 'cpp'
5 self.type = 'c'
6 self.base_url = 'http://en.cppreference.com/w/cpp/'
7 self.root_path = 'header.html'
8
9 html_filters.insert_before 'clean_html', 'c/fix_code'
10 html_filters.push 'cpp/entries', 'c/clean_html', 'title'
11 text_filters.push 'cpp/fix_urls'
12
13 options[:decode_and_clean_paths] = true
14 options[:container] = '#content'
15 options[:title] = false
16 options[:root_title] = 'C++ Programming Language'
17 options[:skip] = %w(
18 language/extending_std.html
19 language/history.html
20 regex/ecmascript.html
21 regex/regex_token_iterator/operator_cmp.html
22 )
23 options[:skip_patterns] = [/experimental/]
24 options[:only_patterns] = [/\.html\z/]
25
26 options[:fix_urls] = ->(url) do
27 url = CGI.unescape(url)
28 url.sub! %r{\A.+/http%3A/}, 'http://'
29 url.sub! 'http://en.cppreference.com/upload.cppreference.com', 'http://upload.cppreference.com'
30 url
31 end
32
33 options[:attribution] = <<-HTML
34 &copy; cppreference.com<br>
35 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
36 HTML
37
38 # Same as get_latest_version in lib/docs/scrapers/c.rb
39 def get_latest_version(opts)
40 doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
41 link = doc.at_css('a[title^="File:"]')
42 date = link.content.scan(/(\d+)\./)[0][0]
43 DateTime.strptime(date, '%Y%m%d').to_time.to_i
44 end
45
46 private
47
48 def file_path_for(*)
49 URI.unescape(super)
50 end
51 end
52end
lib/docs/scrapers/cppref/cpp.rbrenamed from lib/docs/scrapers/cpp20.rb+4−16
@@ -1,17 +1,12 @@
1module Docs1module Docs
2 class Cpp20 < UrlScraper2 class Cpp < Cppref
3 self.name = 'C++20'3 self.name = 'C++'
4 self.slug = 'cpp20'4 self.slug = 'cpp'
5 self.type = 'c'5 self.type = 'c'
6 self.base_url = 'https://en.cppreference.com/w/cpp/'6 self.base_url = 'https://en.cppreference.com/w/cpp/'
7 self.root_path = 'header'
8 7
9 html_filters.insert_before 'clean_html', 'c/fix_code'8 html_filters.insert_before 'cppref/clean_html', 'cpp/entries'
10 html_filters.push 'cpp20/entries', 'c/clean_html', 'title'
11 9
12 options[:decode_and_clean_paths] = true
13 options[:container] = '#content'
14 options[:title] = false
15 options[:root_title] = 'C++ Programming Language'10 options[:root_title] = 'C++ Programming Language'
16 11
17 options[:skip] = %w(12 options[:skip] = %w(
@@ -21,13 +16,6 @@ module Docs · 3 unmodified lines
21 regex/regex_token_iterator/operator_cmp.html16 regex/regex_token_iterator/operator_cmp.html
22 )17 )
23 18
24 options[:skip_patterns] = [/experimental/]
25
26 options[:attribution] = <<-HTML
27 &copy; cppreference.com<br>
28 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
29 HTML
30
31 # Same as get_latest_version in lib/docs/scrapers/c.rb19 # Same as get_latest_version in lib/docs/scrapers/c.rb
32 def get_latest_version(opts)20 def get_latest_version(opts)
33 doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)21 doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
lib/docs/scrapers/cppref/cppref.rb+29−0
@@ -0,0 +1,29 @@
1module Docs
2 class Cppref < UrlScraper
3 self.abstract = true
4 self.type = 'cppref'
5 self.root_path = 'header'
6
7 html_filters.insert_before 'clean_html', 'cppref/fix_code'
8 html_filters.push 'cppref/clean_html', 'title'
9 # 'cpp20/entries',
10 options[:decode_and_clean_paths] = true
11 options[:container] = '#content'
12 options[:title] = false
13 options[:skip] = %w(language/history.html)
14
15 options[:skip_patterns] = [
16 /experimental/
17 ]
18
19 options[:attribution] = <<-HTML
20 &copy; cppreference.com<br>
21 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
22 HTML
23
24 # def get_latest_version
25
26 # end
27
28 end
29end