History

Fix scraper and improve readability

MasterEnoc committed 5 years ago

Commit a8c5c311 parent a4b85692 files · +66 −17Browse files

Unified · Split

2 files
  1. lib/docs/filters/rethinkdb/entries.rb +10 −1
  2. lib/docs/scrapers/rethinkdb.rb +56 −16
lib/docs/filters/rethinkdb/entries.rb+10−1
@@ -1,9 +1,17 @@
1module Docs1module Docs
2 class Rethinkdb2 class Rethinkdb
3 class EntriesFilter < Docs::EntriesFilter3 class EntriesFilter < Docs::EntriesFilter
4
4 def get_name5 def get_name
5 if subpath.start_with?('api')6 if subpath.start_with?('api')
6 at_css('.title').content.remove('ReQL command:').split(', ').first7 name = at_css('.title').content.remove('ReQL command:').split(', ').first
8
9 if name.strip.empty?
10 'lt'
11 else
12 name
13 end
14
7 else15 else
8 at_css('.docs-nav .active').content16 at_css('.docs-nav .active').content
9 end17 end
@@ -30,6 +38,7 @@ module Docs · 20 unmodified lines
30 def include_default_entry?38 def include_default_entry?
31 at_css('.docs-article p').try(:content) != 'Choose your language:'39 at_css('.docs-article p').try(:content) != 'Choose your language:'
32 end40 end
41
33 end42 end
34 end43 end
35end44end
lib/docs/scrapers/rethinkdb.rb+56−16
@@ -13,14 +13,18 @@ module Docs · 12 unmodified lines
13 html_filters.push 'rethinkdb/entries', 'rethinkdb/clean_html'13 html_filters.push 'rethinkdb/entries', 'rethinkdb/clean_html'
14 14
15 options[:trailing_slash] = true15 options[:trailing_slash] = true
16
16 options[:container] = '.documentation'17 options[:container] = '.documentation'
17 18
18 options[:only_patterns] = [/\Adocs/]19 options[:only_patterns] = [/\Adocs/]
20
19 options[:skip_patterns] = [/docs\/install(\-drivers)?\/./]21 options[:skip_patterns] = [/docs\/install(\-drivers)?\/./]
22
20 options[:skip] = %w(23 options[:skip] = %w(
21 docs/build/24 docs/build/
22 docs/tutorials/elections/25 docs/tutorials/elections/
23 docs/tutorials/superheroes/)26 docs/tutorials/superheroes/
27 )
24 28
25 MULTILANG_DOCS = %w(29 MULTILANG_DOCS = %w(
26 changefeeds30 changefeeds
@@ -33,31 +37,66 @@ module Docs · 6 unmodified lines
33 rabbitmq37 rabbitmq
34 secondary-indexes38 secondary-indexes
35 sql-to-reql39 sql-to-reql
36 storing-binary)40 storing-binary
41 )
37 42
38 options[:attribution] = <<-HTML43 options[:attribution] = <<-HTML
39 &copy; RethinkDB contributors<br>44 &copy; RethinkDB contributors<br>
40 Licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported License.45 Licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported License.
41 HTML46 HTML
42 47
43 %w(JavaScript Ruby Python Java).each do |name|48 version 'javascript' do
44 path = name.downcase49 self.initial_paths = %w(api/javascript/)
45 instance_eval <<-CODE
46 version '#{name}' do
47 self.initial_paths = %w(api/#{path}/)
48 50
49 options[:only_patterns] += [/\\Aapi\\/#{path}\\//]51 options[:only_patterns] += [/\Aapi\/javascript\//]
50 52
51 options[:fix_urls] = ->(url) do53 options[:fix_urls] = ->(url) do
52 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})\\z}, 'rethinkdb.com/docs/\\1/#{path}/'54 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})\\z}, 'rethinkdb.com/docs/\\1/javascript/'
53 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})/(?!#{path}/).*}, 'rethinkdb.com/docs/\\1/#{path}/'55 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})/(?!javascript/).*}, 'rethinkdb.com/docs/\\1/javascript/'
54 url.sub! %r{rethinkdb.com/api/(?!javascript|ruby|python|java)}, 'rethinkdb.com/api/#{path}/'56 url.sub! %r{rethinkdb.com/api/(?!javascript|ruby|python|java)}, 'rethinkdb.com/api/javascript/'
55 url57 url
56 end58 end
57 end
58 CODE
59 end59 end
60 60
61 version 'ruby' do
62 self.initial_paths = %w(api/ruby/)
63
64 options[:only_patterns] += [/\Aapi\/ruby\//]
65
66 options[:fix_urls] = ->(url) do
67 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})\\z}, 'rethinkdb.com/docs/\\1/ruby/'
68 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})/(?!ruby/).*}, 'rethinkdb.com/docs/\\1/ruby/'
69 url.sub! %r{rethinkdb.com/api/(?!javascript|ruby|python|java)}, 'rethinkdb.com/api/ruby/'
70 url
71 end
72 end
73
74 version 'python' do
75 self.initial_paths = %w(api/python/)
76
77 options[:only_patterns] += [/\Aapi\/python\//]
78
79 options[:fix_urls] = ->(url) do
80 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})\\z}, 'rethinkdb.com/docs/\\1/python/'
81 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})/(?!python/).*}, 'rethinkdb.com/docs/\\1/python/'
82 url.sub! %r{rethinkdb.com/api/(?!javascript|ruby|python|java)}, 'rethinkdb.com/api/python/'
83 url
84 end
85 end
86
87 version 'java' do
88 self.initial_paths = %w(api/java/)
89
90 options[:only_patterns] += [/\Aapi\/java\//]
91
92 options[:fix_urls] = ->(url) do
93 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})\\z}, 'rethinkdb.com/docs/\\1/java/'
94 url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('|')})/(?!java/).*}, 'rethinkdb.com/docs/\\1/java/'
95 url.sub! %r{rethinkdb.com/api/(?!javascript|ruby|python|java)}, 'rethinkdb.com/api/java/'
96 url
97 end
98 end
99
61 def get_latest_version(opts)100 def get_latest_version(opts)
62 get_latest_github_release('rethinkdb', 'rethinkdb', opts)101 get_latest_github_release('rethinkdb', 'rethinkdb', opts)
63 end102 end
@@ -68,5 +107,6 @@ module Docs · 4 unmodified lines
68 return false unless super107 return false unless super
69 response.body !~ /http-equiv="refresh"/i108 response.body !~ /http-equiv="refresh"/i
70 end109 end
110
71 end111 end
72end112end