History

Initial migration from cpp filescraper to urlscraper

Enoc committed 5 years ago

Commit a754d501 parent b0ab3193 files · +131 −0Browse files

Unified · Split

3 files
  1. lib/docs/filters/cpp20/clean_html.rb +9 −0
  2. lib/docs/filters/cpp20/entries.rb +82 −0
  3. lib/docs/scrapers/cpp20.rb +40 −0
lib/docs/filters/cpp20/clean_html.rb+9−0
@@ -0,0 +1,9 @@
1module Docs
2 class Cpp20
3 class CleanHtmlFilter < Filter
4 def call
5 doc
6 end
7 end
8 end
9end
lib/docs/filters/cpp20/entries.rb+82−0
@@ -0,0 +1,82 @@
1module Docs
2 class Cpp20
3 class EntriesFilter < Docs::EntriesFilter
4 @@duplicate_names = []
5
6 REPLACE_NAMES = {
7 'Error directive' => '#error directive',
8 'Filename and line information' => '#line directive',
9 'Implementation defined behavior control' => '#pragma directive',
10 'Replacing text macros' => '#define directive',
11 'Source file inclusion' => '#include directive' }
12
13 def get_name
14 name = at_css('#firstHeading').content.strip
15 name = format_name(name)
16 name = name.split(',').first
17 name
18 end
19
20 def get_type
21 if at_css('#firstHeading').content.include?('C++ keyword')
22 'Keywords'
23 elsif subpath.start_with?('experimental')
24 'Experimental libraries'
25 elsif subpath.start_with?('language/')
26 'Language'
27 elsif subpath.start_with?('freestanding')
28 'Utilities'
29 elsif type = at_css('.t-navbar > div:nth-child(4) > :first-child').try(:content)
30 type.strip!
31 type.remove! ' library'
32 type.remove! ' utilities'
33 type.remove! 'C++ '
34 type.capitalize!
35 type
36 end
37 end
38
39 def additional_entries
40 return [] if root_page? || self.name.start_with?('operators')
41 names = at_css('#firstHeading').content.remove(%r{\(.+?\)}).split(', ')[1..-1]
42 names.each(&:strip!).reject! do |name|
43 name.size <= 2 || name == '...' || name =~ /\A[<>]/ || name.start_with?('operator')
44 end
45 names.map { |name| [format_name(name)] }
46 end
47
48 def format_name(name)
49 name.remove! 'C++ concepts: '
50 name.remove! 'C++ keywords: '
51 name.remove! 'C++ ' unless name == 'C++ language'
52 name.remove! %r{\s\(.+\)}
53
54 name.sub! %r{\AStandard library header <(.+)>\z}, '\1'
55 name.sub! %r{(<[^>]+>)}, ''
56
57 if name.include?('operator') && name.include?(',')
58 name.sub!(%r{operator.+([\( ])}, 'operators (') || name.sub!(%r{operator.+}, 'operators')
59 name.sub! ' ', ' '
60 name << ')' unless name.last == ')' || name.exclude?('(')
61 name.sub! '()', ''
62 name.sub! %r{\(.+\)}, '' if !name.start_with?('operator') && name.length > 50
63 end
64
65 REPLACE_NAMES[name] || name
66 end
67
68 def entries
69 entries = []
70
71 # avoid duplicate pages
72 if !(@@duplicate_names.include?(name))
73 @@duplicate_names.push(name)
74 entries << default_entry if root_page? || include_default_entry?
75 entries.concat(additional_entries)
76 build_entries(entries)
77 end
78 end
79
80 end
81 end
82end
lib/docs/scrapers/cpp20.rb+40−0
@@ -0,0 +1,40 @@
1module Docs
2 class Cpp20 < UrlScraper
3 self.name = 'C++20'
4 self.slug = 'cpp20'
5 self.type = 'c'
6 self.base_url = 'https://en.cppreference.com/w/cpp/'
7 self.root_path = 'header'
8
9 html_filters.insert_before 'clean_html', 'c/fix_code'
10 html_filters.push 'cpp20/entries', 'c/clean_html', 'title'
11
12 options[:decode_and_clean_paths] = true
13 options[:container] = '#content'
14 options[:title] = false
15 options[:root_title] = 'C++ Programming Language'
16
17 options[:skip] = %w(
18 language/extending_std.html
19 language/history.html
20 regex/ecmascript.html
21 regex/regex_token_iterator/operator_cmp.html
22 )
23
24 options[:skip_patterns] = [/experimental/]
25
26 options[:attribution] = <<-HTML
27 &copy; cppreference.com<br>
28 Licensed under the Creative Commons Attribution-ShareAlike Unported License v3.0.
29 HTML
30
31 # Same as get_latest_version in lib/docs/scrapers/c.rb
32 def get_latest_version(opts)
33 doc = fetch_doc('https://en.cppreference.com/w/Cppreference:Archives', opts)
34 link = doc.at_css('a[title^="File:"]')
35 date = link.content.scan(/(\d+)\./)[0][0]
36 DateTime.strptime(date, '%Y%m%d').to_time.to_i
37 end
38
39 end
40end

Keyboard shortcuts

General

Show keyboard shortcuts
?
Focus the current search
/

Issues and pull requests

Open filters
f

Issues, pull requests and Inbox

Select next item
j↓
Select previous item
k↑
Open selected item
Enter
Dismiss filters or clear selection
Esc

Repository code

Go to file
t