History

Initial migration from cpp filescraper to urlscraper

Enoc committed 5 years ago

Commit a754d501 parent b0ab3193 files · +131 −0Browse files

Unified · Split

3 files
  1. lib/docs/filters/cpp20/clean_html.rb +9 −0
  2. lib/docs/filters/cpp20/entries.rb +82 −0
  3. lib/docs/scrapers/cpp20.rb +40 −0
lib/docs/filters/cpp20/clean_html.rb+9−0
@@ -0,0 +1,9 @@
1+module Docs
2+ class Cpp20
3+ class CleanHtmlFilter < Filter
4+ def call
5+ doc
6+ end
7+ end
8+ end
9+end
lib/docs/filters/cpp20/entries.rb+82−0
@@ -0,0 +1,82 @@
1+module Docs
2+ class Cpp20
3+ class EntriesFilter < Docs::EntriesFilter
4+ @@duplicate_names = []
5+
6+ REPLACE_NAMES = {
7+ 'Error directive' => '#error directive',
8+ 'Filename and line information' => '#line directive',
9+ 'Implementation defined behavior control' => '#pragma directive',
10+ 'Replacing text macros' => '#define directive',
11+ 'Source file inclusion' => '#include directive' }
12+
13+ def get_name
14+ name = at_css('#firstHeading').content.strip
15+ name = format_name(name)
16+ name = name.split(',').first
17+ name
18+ end
19+
20+ def get_type
21+ if at_css('#firstHeading').content.include?('C++ keyword')
22+ 'Keywords'
23+ elsif subpath.start_with?('experimental')
24+ 'Experimental libraries'
25+ elsif subpath.start_with?('language/')
26+ 'Language'
27+ elsif subpath.start_with?('freestanding')
28+ 'Utilities'
29+ elsif type = at_css('.t-navbar > div:nth-child(4) > :first-child').try(:content)
30+ type.strip!
31+ type.remove! ' library'
32+ type.remove! ' utilities'
33+ type.remove! 'C++ '
34+ type.capitalize!
35+ type
36+ end
37+ end
38+
39+ def additional_entries
40+ return [] if root_page? || self.name.start_with?('operators')
41+ names = at_css('#firstHeading').content.remove(%r{\(.+?\)}).split(', ')[1..-1]
42+ names.each(&:strip!).reject! do |name|
43+ name.size <= 2 || name == '...' || name =~ /\A[<>]/ || name.start_with?('operator')
44+ end
45+ names.map { |name| [format_name(name)] }
46+ end
47+
48+ def format_name(name)
49+ name.remove! 'C++ concepts: '
50+ name.remove! 'C++ keywords: '
51+ name.remove! 'C++ ' unless name == 'C++ language'
52+ name.remove! %r{\s\(.+\)}
53+
54+ name.sub! %r{\AStandard library header <(.+)>\z}, '\1'
55+ name.sub! %r{(<[^>]+>)}, ''
56+
57+ if name.include?('operator') && name.include?(',')
58+ name.sub!(%r{operator.+([\( ])}, 'operators (') || name.sub!(%r{operator.+}, 'operators')
59+ name.sub! ' ', ' '
60+ name << ')' unless name.last == ')' || name.exclude?('(')
61+ name.sub! '()', ''
62+ name.sub! %r{\(.+\)}, '' if !name.start_with?('operator') && name.length > 50
63+ end
64+
65+ REPLACE_NAMES[name] || name
66+ end
67+
68+ def entries
69+ entries = []
70+
71+ # avoid duplicate pages
72+ if !(@@duplicate_names.include?(name))
73+ @@duplicate_names.push(name)
74+ entries << default_entry if root_page? || include_default_entry?
75+ entries.concat(additional_entries)
76+ build_entries(entries)
77+ end
78+ end
79+
80+ end
81+ end
82+end
lib/docs/scrapers/cpp20.rb+40−0
@@ -0,0 +1,40 @@
1+module Docs
2+ class Cpp20 < UrlScraper
3+ self.name = 'C++20'
4+ self.slug = 'cpp20'
5+ self.type = 'c'
6+ self.base_url = 'https://en.cppreference.com/w/cpp/'
7+ self.root_path = 'header'
8+
9+ html_filters.insert_before 'clean_html', 'c/fix_code'
10+ html_filters.push 'cpp20/entries', 'c/clean_html', 'title'
11+
12+ options[:decode_and_clean_paths] = true
13+ options[:container] = '#content'
14+ options[:title] = false
15+ options[:root_title] = 'C++ Programming Language'
16+
17+ options[:skip] = %w(
18+ language/extending_std.html
19+ language/history.html
20+ regex/ecmascript.html
21+ regex/regex_token_iterator/operator_cmp.html
22+ )
23+
24+ options[:skip_patterns] = [/experimental/]
25+
26+ options[:attribution] = <<-HTML
27+ &copy; cppreference.com<br>
28+ Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
29+ HTML
30+
31+ # Same as get_latest_version in lib/docs/scrapers/c.rb
32+ def get_latest_version(opts)
33+ doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
34+ link = doc.at_css('a[title^="File:"]')
35+ date = link.content.scan(/(\d+)\./)[0][0]
36+ DateTime.strptime(date, '%Y%m%d').to_time.to_i
37+ end
38+
39+ end
40+end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t