krisyotam/devdocs

lib/docs/filters/cpp/entries.rb+19−1

@@ -1,6 +1,8 @@
1	1	module Docs
2	2	class Cpp
3	3	class EntriesFilter < Docs::EntriesFilter
	4	+ @@duplicate_names = []
	5	+
4	6	REPLACE_NAMES = {
5	7	'Error directive' => '#error directive',
6	8	'Filename and line information' => '#line directive',
@@ -11,7 +13,8 @@ module Docs · 4 unmodified lines
11	13	def get_name
12	14	name = at_css('#firstHeading').content.strip
13	15	name = format_name(name)
14		− name.split(',').first
	16	+ name = name.split(',').first
	17	+ name
15	18	end
16	19
17	20	def get_type
@@ -61,6 +64,21 @@ module Docs · 43 unmodified lines
61	64
62	65	REPLACE_NAMES[name] \|\| name
63	66	end
	67	+
	68	+ # Avoid duplicate pages, these duplicate page are the same page for
	69	+ # multiple functions that are organized in the same page because provide
	70	+ # similar behavior but have different name.
	71	+ def entries
	72	+ entries = []
	73	+
	74	+ if !(@@duplicate_names.include?(name))
	75	+ @@duplicate_names.push(name)
	76	+ entries << default_entry if root_page? \|\| include_default_entry?
	77	+ entries.concat(additional_entries)
	78	+ build_entries(entries)
	79	+ end
	80	+ end
	81	+
64	82	end
65	83	end
66	84	end

lib/docs/filters/cpp/fix_urls.rb+0−11

@@ -1,11 +0,0 @@
1		−module Docs
2		− class Cpp
3		− class FixUrlsFilter < Filter
4		− def call
5		− html.gsub! File.join(Cpp.base_url, Cpp.root_path), Cpp.base_url[0..-2]
6		− html.gsub! %r{#{Cpp.base_url}([^"']+?)\.html}, "#{Cpp.base_url}\\1"
7		− html
8		− end
9		− end
10		− end
11		−end

lib/docs/filters/cpp20/clean_html.rb+0−9

@@ -1,9 +0,0 @@
1		−module Docs
2		− class Cpp20
3		− class CleanHtmlFilter < Filter
4		− def call
5		− doc
6		− end
7		− end
8		− end
9		−end

lib/docs/filters/cpp20/entries.rb+0−82

@@ -1,82 +0,0 @@
1		−module Docs
2		− class Cpp20
3		− class EntriesFilter < Docs::EntriesFilter
4		− @@duplicate_names = []
5		−
6		− REPLACE_NAMES = {
7		− 'Error directive' => '#error directive',
8		− 'Filename and line information' => '#line directive',
9		− 'Implementation defined behavior control' => '#pragma directive',
10		− 'Replacing text macros' => '#define directive',
11		− 'Source file inclusion' => '#include directive' }
12		−
13		− def get_name
14		− name = at_css('#firstHeading').content.strip
15		− name = format_name(name)
16		− name = name.split(',').first
17		− name
18		− end
19		−
20		− def get_type
21		− if at_css('#firstHeading').content.include?('C++ keyword')
22		− 'Keywords'
23		− elsif subpath.start_with?('experimental')
24		− 'Experimental libraries'
25		− elsif subpath.start_with?('language/')
26		− 'Language'
27		− elsif subpath.start_with?('freestanding')
28		− 'Utilities'
29		− elsif type = at_css('.t-navbar > div:nth-child(4) > :first-child').try(:content)
30		− type.strip!
31		− type.remove! ' library'
32		− type.remove! ' utilities'
33		− type.remove! 'C++ '
34		− type.capitalize!
35		− type
36		− end
37		− end
38		−
39		− def additional_entries
40		− return [] if root_page? \|\| self.name.start_with?('operators')
41		− names = at_css('#firstHeading').content.remove(%r{\(.+?\)}).split(', ')[1..-1]
42		− names.each(&:strip!).reject! do \|name\|
43		− name.size <= 2 \|\| name == '...' \|\| name =~ /\A[<>]/ \|\| name.start_with?('operator')
44		− end
45		− names.map { \|name\| [format_name(name)] }
46		− end
47		−
48		− def format_name(name)
49		− name.remove! 'C++ concepts: '
50		− name.remove! 'C++ keywords: '
51		− name.remove! 'C++ ' unless name == 'C++ language'
52		− name.remove! %r{\s\(.+\)}
53		−
54		− name.sub! %r{\AStandard library header <(.+)>\z}, '\1'
55		− name.sub! %r{(<[^>]+>)}, ''
56		−
57		− if name.include?('operator') && name.include?(',')
58		− name.sub!(%r{operator.+([\( ])}, 'operators (') \|\| name.sub!(%r{operator.+}, 'operators')
59		− name.sub! ' ', ' '
60		− name << ')' unless name.last == ')' \|\| name.exclude?('(')
61		− name.sub! '()', ''
62		− name.sub! %r{\(.+\)}, '' if !name.start_with?('operator') && name.length > 50
63		− end
64		−
65		− REPLACE_NAMES[name] \|\| name
66		− end
67		−
68		− def entries
69		− entries = []
70		−
71		− # avoid duplicate pages
72		− if !(@@duplicate_names.include?(name))
73		− @@duplicate_names.push(name)
74		− entries << default_entry if root_page? \|\| include_default_entry?
75		− entries.concat(additional_entries)
76		− build_entries(entries)
77		− end
78		− end
79		−
80		− end
81		− end
82		−end

lib/docs/filters/cppref/clean_html.rb+116−0

@@ -0,0 +1,116 @@
	1	+module Docs
	2	+ class Cppref
	3	+ class CleanHtmlFilter < Filter
	4	+ def call
	5	+ css('h1').remove if root_page?
	6	+
	7	+ css('.t-dcl-rev-aux td[rowspan]').each do \|node\|
	8	+ rowspan = node['rowspan'].to_i
	9	+ node['rowspan'] = node.ancestors('tbody').css('tr').length if rowspan > 3
	10	+ end
	11	+
	12	+ css('#siteSub', '#contentSub', '.printfooter', '.t-navbar', '.editsection', '#toc',
	13	+ '.t-dsc-sep', '.t-dcl-sep', '#catlinks', '.ambox-notice', '.mw-cite-backlink',
	14	+ '.t-sdsc-sep:first-child:last-child', '.t-example-live-link',
	15	+ '.t-dcl-rev-num > .t-dcl-rev-aux ~ tr:not(.t-dcl-rev-aux) > td:nth-child(2)').remove
	16	+
	17	+ css('#bodyContent', '.mw-content-ltr', 'span[style]', 'div[class^="t-ref"]', '.t-image',
	18	+ 'th > div', 'td > div', '.t-dsc-see', '.mainpagediv', 'code > b', 'tbody').each do \|node\|
	19	+ node.before(node.children).remove
	20	+ end
	21	+
	22	+ css('div > ul').each do \|node\|
	23	+ node.parent.before(node.parent.children).remove
	24	+ end
	25	+
	26	+ css('dl > dd:first-child:last-child > ul:first-child:last-child').each do \|node\|
	27	+ dl = node.parent.parent
	28	+ if dl.previous_element && dl.previous_element.name == 'ul'
	29	+ dl.previous_element << node
	30	+ dl.remove
	31	+ else
	32	+ dl.before(node).remove
	33	+ end
	34	+ end
	35	+
	36	+ css('dl > dd:first-child:last-child').each do \|node\|
	37	+ node.parent.before(node.children).remove
	38	+ end
	39	+
	40	+ css('ul').each do \|node\|
	41	+ while node.next_element && node.next_element.name == 'ul'
	42	+ node << node.next_element.children
	43	+ node.next_element.remove
	44	+ end
	45	+ end
	46	+
	47	+ css('h2 > span[id]', 'h3 > span[id]', 'h4 > span[id]', 'h5 > span[id]', 'h6 > span[id]').each do \|node\|
	48	+ node.parent['id'] = node['id']
	49	+ node.before(node.children).remove
	50	+ end
	51	+
	52	+ css('table[style]', 'th[style]', 'td[style]').remove_attr('style')
	53	+ css('table[cellpadding]').remove_attr('cellpadding')
	54	+
	55	+ css('.t-dsc-hitem > td', '.t-dsc-header > td').each do \|node\|
	56	+ node.name = 'th'
	57	+ node.content = ' ' if node.content.empty?
	58	+ end
	59	+
	60	+ css('tt', 'span > span.source-cpp', 'span.t-c', 'span.t-lc', 'span.t-dsc-see-tt').each do \|node\|
	61	+ node.name = 'code'
	62	+ node.remove_attribute('class')
	63	+ node.content = node.content unless node.at_css('a')
	64	+ end
	65	+
	66	+ css('div > span.source-cpp').each do \|node\|
	67	+ node.name = 'pre'
	68	+ node.inner_html = node.inner_html.gsub('<br>', "\n")
	69	+ node.content = node.content
	70	+ end
	71	+
	72	+ css('div > a > img[alt="About this image"]').each do \|node\|
	73	+ node.parent.parent.remove
	74	+ end
	75	+
	76	+ css('area[href]').each do \|node\|
	77	+ node['href'] = node['href'].remove('.html')
	78	+ end
	79	+
	80	+ css('p').each do \|node\|
	81	+ while node.next && (node.next.text? \|\| node.next.name == 'a' \|\| node.next.name == 'code')
	82	+ node << node.next
	83	+ end
	84	+ node.inner_html = node.inner_html.strip
	85	+ node << '.' if node.content =~ /[a-zA-Z0-9\)]\z/
	86	+ node.remove if node.content.blank? && !node.at_css('img')
	87	+ end
	88	+
	89	+ css('pre').each do \|node\|
	90	+ node['data-language'] = if node['class'].try(:include?, 'cpp') \|\| node.parent['class'].try(:include?, 'cpp')
	91	+ 'cpp'
	92	+ else
	93	+ 'c'
	94	+ end
	95	+ node.remove_attribute('class')
	96	+ node.content = node.content.gsub("\t", ' ' * 8)
	97	+ end
	98	+
	99	+ css('code code', '.mw-geshi').each do \|node\|
	100	+ node.before(node.children).remove
	101	+ end
	102	+
	103	+ css('h1 ~ .fmbox').each do \|node\|
	104	+ node.name = 'div'
	105	+ node.content = node.content
	106	+ end
	107	+
	108	+ css('img').each do \|node\|
	109	+ node['src'] = node['src'].sub! %r{http://en.cppreference.com/common/([^"']+?)\.svg}, 'http://upload.cppreference.com/mwiki/\1.svg'
	110	+ end
	111	+
	112	+ doc
	113	+ end
	114	+ end
	115	+ end
	116	+end

lib/docs/filters/cppref/fix_code.rb+21−0

@@ -0,0 +1,21 @@
	1	+module Docs
	2	+ class Cppref
	3	+ class FixCodeFilter < Filter
	4	+ def call
	5	+ css('div > span.source-c', 'div > span.source-cpp').each do \|node\|
	6	+ node.inner_html = node.inner_html.gsub(/<br>\n?/, "\n").gsub("\n</p>\n", "</p>\n")
	7	+ node.parent.name = 'pre'
	8	+ node.parent['class'] = node['class']
	9	+ node.parent.content = node.content
	10	+ end
	11	+
	12	+ nbsp = Nokogiri::HTML(' ').text
	13	+ css('pre').each do \|node\|
	14	+ node.content = node.content.gsub(nbsp, ' ')
	15	+ end
	16	+
	17	+ doc
	18	+ end
	19	+ end
	20	+ end
	21	+end

lib/docs/scrapers/cpp.rb+0−52

@@ -1,52 +0,0 @@
1		−module Docs
2		− class Cpp < FileScraper
3		− self.name = 'C++'
4		− self.slug = 'cpp'
5		− self.type = 'c'
6		− self.base_url = 'http://en.cppreference.com/w/cpp/'
7		− self.root_path = 'header.html'
8		−
9		− html_filters.insert_before 'clean_html', 'c/fix_code'
10		− html_filters.push 'cpp/entries', 'c/clean_html', 'title'
11		− text_filters.push 'cpp/fix_urls'
12		−
13		− options[:decode_and_clean_paths] = true
14		− options[:container] = '#content'
15		− options[:title] = false
16		− options[:root_title] = 'C++ Programming Language'
17		− options[:skip] = %w(
18		− language/extending_std.html
19		− language/history.html
20		− regex/ecmascript.html
21		− regex/regex_token_iterator/operator_cmp.html
22		− )
23		− options[:skip_patterns] = [/experimental/]
24		− options[:only_patterns] = [/\.html\z/]
25		−
26		− options[:fix_urls] = ->(url) do
27		− url = CGI.unescape(url)
28		− url.sub! %r{\A.+/http%3A/}, 'http://'
29		− url.sub! 'http://en.cppreference.com/upload.cppreference.com', 'http://upload.cppreference.com'
30		− url
31		− end
32		−
33		− options[:attribution] = <<-HTML
34		− © cppreference.com<br>
35		− Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
36		− HTML
37		−
38		− # Same as get_latest_version in lib/docs/scrapers/c.rb
39		− def get_latest_version(opts)
40		− doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
41		− link = doc.at_css('a[title^="File:"]')
42		− date = link.content.scan(/(\d+)\./)[0][0]
43		− DateTime.strptime(date, '%Y%m%d').to_time.to_i
44		− end
45		−
46		− private
47		−
48		− def file_path_for(*)
49		− URI.unescape(super)
50		− end
51		− end
52		−end

lib/docs/scrapers/cppref/cpp.rbrenamed from lib/docs/scrapers/cpp20.rb+4−16

@@ -1,17 +1,12 @@
1	1	module Docs
2		− class Cpp20 < UrlScraper
3		− self.name = 'C++20'
4		− self.slug = 'cpp20'
	2	+ class Cpp < Cppref
	3	+ self.name = 'C++'
	4	+ self.slug = 'cpp'
5	5	self.type = 'c'
6	6	self.base_url = 'https://en.cppreference.com/w/cpp/'
7		− self.root_path = 'header'
8	7
9		− html_filters.insert_before 'clean_html', 'c/fix_code'
10		− html_filters.push 'cpp20/entries', 'c/clean_html', 'title'
	8	+ html_filters.insert_before 'cppref/clean_html', 'cpp/entries'
11	9
12		− options[:decode_and_clean_paths] = true
13		− options[:container] = '#content'
14		− options[:title] = false
15	10	options[:root_title] = 'C++ Programming Language'
16	11
17	12	options[:skip] = %w(
@@ -21,13 +16,6 @@ module Docs · 3 unmodified lines
21	16	regex/regex_token_iterator/operator_cmp.html
22	17	)
23	18
24		− options[:skip_patterns] = [/experimental/]
25		−
26		− options[:attribution] = <<-HTML
27		− © cppreference.com<br>
28		− Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
29		− HTML
30		−
31	19	# Same as get_latest_version in lib/docs/scrapers/c.rb
32	20	def get_latest_version(opts)
33	21	doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)

lib/docs/scrapers/cppref/cppref.rb+29−0

@@ -0,0 +1,29 @@
	1	+module Docs
	2	+ class Cppref < UrlScraper
	3	+ self.abstract = true
	4	+ self.type = 'cppref'
	5	+ self.root_path = 'header'
	6	+
	7	+ html_filters.insert_before 'clean_html', 'cppref/fix_code'
	8	+ html_filters.push 'cppref/clean_html', 'title'
	9	+ # 'cpp20/entries',
	10	+ options[:decode_and_clean_paths] = true
	11	+ options[:container] = '#content'
	12	+ options[:title] = false
	13	+ options[:skip] = %w(language/history.html)
	14	+
	15	+ options[:skip_patterns] = [
	16	+ /experimental/
	17	+ ]
	18	+
	19	+ options[:attribution] = <<-HTML
	20	+ © cppreference.com<br>
	21	+ Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
	22	+ HTML
	23	+
	24	+ # def get_latest_version
	25	+
	26	+ # end
	27	+
	28	+ end
	29	+end