History

migrate cpp scraper from filescraper to urlscraper

Enoc committed 5 years ago

Commit e17bc841 parent a754d509 files · +189 −171Browse files

Showing the changes to lib/docs/filters/cppref (2 of 9 files). Show all

Unified · Split

2 files
  1. lib/docs/filters/cppref/clean_html.rb +116 −0
  2. lib/docs/filters/cppref/fix_code.rb +21 −0
lib/docs/filters/cppref/clean_html.rb+116−0
@@ -0,0 +1,116 @@
1+module Docs
2+ class Cppref
3+ class CleanHtmlFilter < Filter
4+ def call
5+ css('h1').remove if root_page?
6+
7+ css('.t-dcl-rev-aux td[rowspan]').each do |node|
8+ rowspan = node['rowspan'].to_i
9+ node['rowspan'] = node.ancestors('tbody').css('tr').length if rowspan > 3
10+ end
11+
12+ css('#siteSub', '#contentSub', '.printfooter', '.t-navbar', '.editsection', '#toc',
13+ '.t-dsc-sep', '.t-dcl-sep', '#catlinks', '.ambox-notice', '.mw-cite-backlink',
14+ '.t-sdsc-sep:first-child:last-child', '.t-example-live-link',
15+ '.t-dcl-rev-num > .t-dcl-rev-aux ~ tr:not(.t-dcl-rev-aux) > td:nth-child(2)').remove
16+
17+ css('#bodyContent', '.mw-content-ltr', 'span[style]', 'div[class^="t-ref"]', '.t-image',
18+ 'th > div', 'td > div', '.t-dsc-see', '.mainpagediv', 'code > b', 'tbody').each do |node|
19+ node.before(node.children).remove
20+ end
21+
22+ css('div > ul').each do |node|
23+ node.parent.before(node.parent.children).remove
24+ end
25+
26+ css('dl > dd:first-child:last-child > ul:first-child:last-child').each do |node|
27+ dl = node.parent.parent
28+ if dl.previous_element && dl.previous_element.name == 'ul'
29+ dl.previous_element << node
30+ dl.remove
31+ else
32+ dl.before(node).remove
33+ end
34+ end
35+
36+ css('dl > dd:first-child:last-child').each do |node|
37+ node.parent.before(node.children).remove
38+ end
39+
40+ css('ul').each do |node|
41+ while node.next_element && node.next_element.name == 'ul'
42+ node << node.next_element.children
43+ node.next_element.remove
44+ end
45+ end
46+
47+ css('h2 > span[id]', 'h3 > span[id]', 'h4 > span[id]', 'h5 > span[id]', 'h6 > span[id]').each do |node|
48+ node.parent['id'] = node['id']
49+ node.before(node.children).remove
50+ end
51+
52+ css('table[style]', 'th[style]', 'td[style]').remove_attr('style')
53+ css('table[cellpadding]').remove_attr('cellpadding')
54+
55+ css('.t-dsc-hitem > td', '.t-dsc-header > td').each do |node|
56+ node.name = 'th'
57+ node.content = ' ' if node.content.empty?
58+ end
59+
60+ css('tt', 'span > span.source-cpp', 'span.t-c', 'span.t-lc', 'span.t-dsc-see-tt').each do |node|
61+ node.name = 'code'
62+ node.remove_attribute('class')
63+ node.content = node.content unless node.at_css('a')
64+ end
65+
66+ css('div > span.source-cpp').each do |node|
67+ node.name = 'pre'
68+ node.inner_html = node.inner_html.gsub('<br>', "\n")
69+ node.content = node.content
70+ end
71+
72+ css('div > a > img[alt="About this image"]').each do |node|
73+ node.parent.parent.remove
74+ end
75+
76+ css('area[href]').each do |node|
77+ node['href'] = node['href'].remove('.html')
78+ end
79+
80+ css('p').each do |node|
81+ while node.next && (node.next.text? || node.next.name == 'a' || node.next.name == 'code')
82+ node << node.next
83+ end
84+ node.inner_html = node.inner_html.strip
85+ node << '.' if node.content =~ /[a-zA-Z0-9\)]\z/
86+ node.remove if node.content.blank? && !node.at_css('img')
87+ end
88+
89+ css('pre').each do |node|
90+ node['data-language'] = if node['class'].try(:include?, 'cpp') || node.parent['class'].try(:include?, 'cpp')
91+ 'cpp'
92+ else
93+ 'c'
94+ end
95+ node.remove_attribute('class')
96+ node.content = node.content.gsub("\t", ' ' * 8)
97+ end
98+
99+ css('code code', '.mw-geshi').each do |node|
100+ node.before(node.children).remove
101+ end
102+
103+ css('h1 ~ .fmbox').each do |node|
104+ node.name = 'div'
105+ node.content = node.content
106+ end
107+
108+ css('img').each do |node|
109+ node['src'] = node['src'].sub! %r{http://en.cppreference.com/common/([^"']+?)\.svg}, 'http://upload.cppreference.com/mwiki/\1.svg'
110+ end
111+
112+ doc
113+ end
114+ end
115+ end
116+end
lib/docs/filters/cppref/fix_code.rb+21−0
@@ -0,0 +1,21 @@
1+module Docs
2+ class Cppref
3+ class FixCodeFilter < Filter
4+ def call
5+ css('div > span.source-c', 'div > span.source-cpp').each do |node|
6+ node.inner_html = node.inner_html.gsub(/<br>\n?/, "\n").gsub("\n</p>\n", "</p>\n")
7+ node.parent.name = 'pre'
8+ node.parent['class'] = node['class']
9+ node.parent.content = node.content
10+ end
11+
12+ nbsp = Nokogiri::HTML('&nbsp;').text
13+ css('pre').each do |node|
14+ node.content = node.content.gsub(nbsp, ' ')
15+ end
16+
17+ doc
18+ end
19+ end
20+ end
21+end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t