History

Migrate c scraper from filescraper to urlscraper

Enoc committed 5 years ago

Commit fdfcf3d1 parent e17bc8410 files · +24 −204Browse files

Unified · Split

10 files
  1. assets/stylesheets/application.css.scss +1 −1
  2. assets/stylesheets/pages/_cppref.scss +0 −0
  3. lib/docs/filters/c/clean_html.rb +0 −116
  4. lib/docs/filters/c/entries.rb +3 −0
  5. lib/docs/filters/c/fix_code.rb +0 −21
  6. lib/docs/filters/c/fix_urls.rb +0 −11
  7. lib/docs/scrapers/c.rb +0 −42
  8. lib/docs/scrapers/cppref/c.rb +12 −0
  9. lib/docs/scrapers/cppref/cpp.rb +0 −9
  10. lib/docs/scrapers/cppref/cppref.rb +8 −4
assets/stylesheets/application.css.scss+1−1
@@ -39,7 +39,7 @@ · 38 unmodified lines
3939 'pages/async',
4040 'pages/bash',
4141 'pages/bootstrap',
42− 'pages/c',
42+ 'pages/cppref',
4343 'pages/cakephp',
4444 'pages/clojure',
4545 'pages/codeception',
assets/stylesheets/pages/_cppref.scssrenamed from assets/stylesheets/pages/_c.scss+0−0

File renamed without changes.

lib/docs/filters/c/clean_html.rb+0−116
@@ -1,116 +0,0 @@
1−module Docs
2− class C
3− class CleanHtmlFilter < Filter
4− def call
5− css('h1').remove if root_page?
6−
7− css('.t-dcl-rev-aux td[rowspan]').each do |node|
8− rowspan = node['rowspan'].to_i
9− node['rowspan'] = node.ancestors('tbody').css('tr').length if rowspan > 3
10− end
11−
12− css('#siteSub', '#contentSub', '.printfooter', '.t-navbar', '.editsection', '#toc',
13− '.t-dsc-sep', '.t-dcl-sep', '#catlinks', '.ambox-notice', '.mw-cite-backlink',
14− '.t-sdsc-sep:first-child:last-child', '.t-example-live-link',
15− '.t-dcl-rev-num > .t-dcl-rev-aux ~ tr:not(.t-dcl-rev-aux) > td:nth-child(2)').remove
16−
17− css('#bodyContent', '.mw-content-ltr', 'span[style]', 'div[class^="t-ref"]', '.t-image',
18− 'th > div', 'td > div', '.t-dsc-see', '.mainpagediv', 'code > b', 'tbody').each do |node|
19− node.before(node.children).remove
20− end
21−
22− css('div > ul').each do |node|
23− node.parent.before(node.parent.children).remove
24− end
25−
26− css('dl > dd:first-child:last-child > ul:first-child:last-child').each do |node|
27− dl = node.parent.parent
28− if dl.previous_element && dl.previous_element.name == 'ul'
29− dl.previous_element << node
30− dl.remove
31− else
32− dl.before(node).remove
33− end
34− end
35−
36− css('dl > dd:first-child:last-child').each do |node|
37− node.parent.before(node.children).remove
38− end
39−
40− css('ul').each do |node|
41− while node.next_element && node.next_element.name == 'ul'
42− node << node.next_element.children
43− node.next_element.remove
44− end
45− end
46−
47− css('h2 > span[id]', 'h3 > span[id]', 'h4 > span[id]', 'h5 > span[id]', 'h6 > span[id]').each do |node|
48− node.parent['id'] = node['id']
49− node.before(node.children).remove
50− end
51−
52− css('table[style]', 'th[style]', 'td[style]').remove_attr('style')
53− css('table[cellpadding]').remove_attr('cellpadding')
54−
55− css('.t-dsc-hitem > td', '.t-dsc-header > td').each do |node|
56− node.name = 'th'
57− node.content = ' ' if node.content.empty?
58− end
59−
60− css('tt', 'span > span.source-cpp', 'span.t-c', 'span.t-lc', 'span.t-dsc-see-tt').each do |node|
61− node.name = 'code'
62− node.remove_attribute('class')
63− node.content = node.content unless node.at_css('a')
64− end
65−
66− css('div > span.source-cpp').each do |node|
67− node.name = 'pre'
68− node.inner_html = node.inner_html.gsub('<br>', "\n")
69− node.content = node.content
70− end
71−
72− css('div > a > img[alt="About this image"]').each do |node|
73− node.parent.parent.remove
74− end
75−
76− css('area[href]').each do |node|
77− node['href'] = node['href'].remove('.html')
78− end
79−
80− css('p').each do |node|
81− while node.next && (node.next.text? || node.next.name == 'a' || node.next.name == 'code')
82− node << node.next
83− end
84− node.inner_html = node.inner_html.strip
85− node << '.' if node.content =~ /[a-zA-Z0-9\)]\z/
86− node.remove if node.content.blank? && !node.at_css('img')
87− end
88−
89− css('pre').each do |node|
90− node['data-language'] = if node['class'].try(:include?, 'cpp') || node.parent['class'].try(:include?, 'cpp')
91− 'cpp'
92− else
93− 'c'
94− end
95− node.remove_attribute('class')
96− node.content = node.content.gsub("\t", ' ' * 8)
97− end
98−
99− css('code code', '.mw-geshi').each do |node|
100− node.before(node.children).remove
101− end
102−
103− css('h1 ~ .fmbox').each do |node|
104− node.name = 'div'
105− node.content = node.content
106− end
107−
108− css('img').each do |node|
109− node['src'] = node['src'].sub! %r{http://en.cppreference.com/common/([^"']+?)\.svg}, 'http://upload.cppreference.com/mwiki/\1.svg'
110− end
111−
112− doc
113− end
114− end
115− end
116−end
lib/docs/filters/c/entries.rb+3−0
@@ -22,6 +22,9 @@ module Docs · 21 unmodified lines
2222 end
2323
2424 def get_type
25+
26+ return "C keywords" if slug =~ /keyword/
27+
2528 type = at_css('.t-navbar > div:nth-child(4) > :first-child').try(:content)
2629 type.strip!
2730 type.remove! ' library'
lib/docs/filters/c/fix_code.rb+0−21
@@ -1,21 +0,0 @@
1−module Docs
2− class C
3− class FixCodeFilter < Filter
4− def call
5− css('div > span.source-c', 'div > span.source-cpp').each do |node|
6− node.inner_html = node.inner_html.gsub(/<br>\n?/, "\n").gsub("\n</p>\n", "</p>\n")
7− node.parent.name = 'pre'
8− node.parent['class'] = node['class']
9− node.parent.content = node.content
10− end
11−
12− nbsp = Nokogiri::HTML('&nbsp;').text
13− css('pre').each do |node|
14− node.content = node.content.gsub(nbsp, ' ')
15− end
16−
17− doc
18− end
19− end
20− end
21−end
lib/docs/filters/c/fix_urls.rb+0−11
@@ -1,11 +0,0 @@
1−module Docs
2− class C
3− class FixUrlsFilter < Filter
4− def call
5− html.gsub! File.join(C.base_url, C.root_path), C.base_url[0..-2]
6− html.gsub! %r{#{C.base_url}([^"']+?)\.html}, "#{C.base_url}\\1"
7− html
8− end
9− end
10− end
11−end
lib/docs/scrapers/c.rb+0−42
@@ -1,42 +0,0 @@
1−module Docs
2− class C < FileScraper
3− self.type = 'c'
4− self.base_url = 'http://en.cppreference.com/w/c/'
5− self.root_path = 'header.html'
6−
7− html_filters.insert_before 'clean_html', 'c/fix_code'
8− html_filters.push 'c/entries', 'c/clean_html', 'title'
9− text_filters.push 'c/fix_urls'
10−
11− options[:decode_and_clean_paths] = true
12− options[:container] = '#content'
13− options[:title] = false
14− options[:root_title] = 'C Programming Language'
15− options[:skip] = %w(language/history.html)
16− options[:skip_patterns] = [/experimental/]
17−
18− options[:fix_urls] = ->(url) do
19− url.sub! %r{\A.+/http%3A/}, 'http://'
20− url.sub! 'http://en.cppreference.com/upload.cppreference.com', 'http://upload.cppreference.com'
21− url
22− end
23−
24− options[:attribution] = <<-HTML
25− &copy; cppreference.com<br>
26− Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
27− HTML
28−
29− def get_latest_version(opts)
30− doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
31− link = doc.at_css('a[title^="File:"]')
32− date = link.content.scan(/(\d+)\./)[0][0]
33− DateTime.strptime(date, '%Y%m%d').to_time.to_i
34− end
35−
36− private
37−
38− def file_path_for(*)
39− URI.unescape(super)
40− end
41− end
42−end
lib/docs/scrapers/cppref/c.rb+12−0
@@ -0,0 +1,12 @@
1+module Docs
2+ class C < Cppref
3+ self.name = 'c'
4+ self.slug = 'c'
5+ self.base_url = 'https://en.cppreference.com/w/c/'
6+
7+ html_filters.insert_before 'cppref/clean_html', 'c/entries'
8+
9+ options[:root_title] = 'C Programming Language'
10+
11+ end
12+end
lib/docs/scrapers/cppref/cpp.rb+0−9
@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
22 class Cpp < Cppref
33 self.name = 'C++'
44 self.slug = 'cpp'
5− self.type = 'c'
65 self.base_url = 'https://en.cppreference.com/w/cpp/'
76
87 html_filters.insert_before 'cppref/clean_html', 'cpp/entries'
@@ -16,13 +15,5 @@ module Docs · 7 unmodified lines
1615 regex/regex_token_iterator/operator_cmp.html
1716 )
1817
19− # Same as get_latest_version in lib/docs/scrapers/c.rb
20− def get_latest_version(opts)
21− doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
22− link = doc.at_css('a[title^="File:"]')
23− date = link.content.scan(/(\d+)\./)[0][0]
24− DateTime.strptime(date, '%Y%m%d').to_time.to_i
25− end
26−
2718 end
2819 end
lib/docs/scrapers/cppref/cppref.rb+8−4
@@ -6,7 +6,7 @@ module Docs · 5 unmodified lines
66
77 html_filters.insert_before 'clean_html', 'cppref/fix_code'
88 html_filters.push 'cppref/clean_html', 'title'
9− # 'cpp20/entries',
9+
1010 options[:decode_and_clean_paths] = true
1111 options[:container] = '#content'
1212 options[:title] = false
@@ -21,9 +21,13 @@ module Docs · 8 unmodified lines
2121 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
2222 HTML
2323
24− # def get_latest_version
25−
26− # end
24+ # Check if the 'headers' page has changed
25+ def get_latest_version(opts)
26+ doc = fetch_doc(self.base_url + self.root_path, opts)
27+ date = doc.at_css('#footer-info-lastmod').content
28+ date = date.match(/[[:digit:]]{1,2} .* [[:digit:]]{4}/).to_s
29+ date = DateTime.strptime(date, '%e %B %Y').to_time.to_i
30+ end
2731
2832 end
2933 end