Fix scraper and improve readability

MasterEnoc committed 5 years ago

Commit a8c5c311 parent a4b85692 files · +66 −17Browse files

Unified · Split

2 files

lib/docs/filters/rethinkdb/entries.rb +10 −1
lib/docs/scrapers/rethinkdb.rb +56 −16

lib/docs/filters/rethinkdb/entries.rb+10−1

@@ -1,9 +1,17 @@
1	1	module Docs
2	2	class Rethinkdb
3	3	class EntriesFilter < Docs::EntriesFilter
	4	+
4	5	def get_name
5	6	if subpath.start_with?('api')
6		− at_css('.title').content.remove('ReQL command:').split(', ').first
	7	+ name = at_css('.title').content.remove('ReQL command:').split(', ').first
	8	+
	9	+ if name.strip.empty?
	10	+ 'lt'
	11	+ else
	12	+ name
	13	+ end
	14	+
7	15	else
8	16	at_css('.docs-nav .active').content
9	17	end
@@ -30,6 +38,7 @@ module Docs · 20 unmodified lines
30	38	def include_default_entry?
31	39	at_css('.docs-article p').try(:content) != 'Choose your language:'
32	40	end
	41	+
33	42	end
34	43	end
35	44	end

lib/docs/scrapers/rethinkdb.rb+56−16

@@ -13,14 +13,18 @@ module Docs · 12 unmodified lines
13	13	html_filters.push 'rethinkdb/entries', 'rethinkdb/clean_html'
14	14
15	15	options[:trailing_slash] = true
	16	+
16	17	options[:container] = '.documentation'
17	18
18	19	options[:only_patterns] = [/\Adocs/]
	20	+
19	21	options[:skip_patterns] = [/docs\/install(\-drivers)?\/./]
	22	+
20	23	options[:skip] = %w(
21	24	docs/build/
22	25	docs/tutorials/elections/
23		− docs/tutorials/superheroes/)
	26	+ docs/tutorials/superheroes/
	27	+ )
24	28
25	29	MULTILANG_DOCS = %w(
26	30	changefeeds
@@ -33,31 +37,66 @@ module Docs · 6 unmodified lines
33	37	rabbitmq
34	38	secondary-indexes
35	39	sql-to-reql
36		− storing-binary)
	40	+ storing-binary
	41	+ )
37	42
38	43	options[:attribution] = <<-HTML
39	44	© RethinkDB contributors<br>
40	45	Licensed under the Creative Commons Attribution-ShareAlike 3.0 Unported License.
41	46	HTML
42	47
43		− %w(JavaScript Ruby Python Java).each do \|name\|
44		− path = name.downcase
45		− instance_eval <<-CODE
46		− version '#{name}' do
47		− self.initial_paths = %w(api/#{path}/)
	48	+ version 'javascript' do
	49	+ self.initial_paths = %w(api/javascript/)
48	50
49		− options[:only_patterns] += [/\\Aapi\\/#{path}\\//]
	51	+ options[:only_patterns] += [/\Aapi\/javascript\//]
50	52
51		− options[:fix_urls] = ->(url) do
52		− url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})\\z}, 'rethinkdb.com/docs/\\1/#{path}/'
53		− url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})/(?!#{path}/).*}, 'rethinkdb.com/docs/\\1/#{path}/'
54		− url.sub! %r{rethinkdb.com/api/(?!javascript\|ruby\|python\|java)}, 'rethinkdb.com/api/#{path}/'
55		− url
56		− end
57		− end
58		− CODE
	53	+ options[:fix_urls] = ->(url) do
	54	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})\\z}, 'rethinkdb.com/docs/\\1/javascript/'
	55	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})/(?!javascript/).*}, 'rethinkdb.com/docs/\\1/javascript/'
	56	+ url.sub! %r{rethinkdb.com/api/(?!javascript\|ruby\|python\|java)}, 'rethinkdb.com/api/javascript/'
	57	+ url
	58	+ end
59	59	end
60	60
	61	+ version 'ruby' do
	62	+ self.initial_paths = %w(api/ruby/)
	63	+
	64	+ options[:only_patterns] += [/\Aapi\/ruby\//]
	65	+
	66	+ options[:fix_urls] = ->(url) do
	67	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})\\z}, 'rethinkdb.com/docs/\\1/ruby/'
	68	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})/(?!ruby/).*}, 'rethinkdb.com/docs/\\1/ruby/'
	69	+ url.sub! %r{rethinkdb.com/api/(?!javascript\|ruby\|python\|java)}, 'rethinkdb.com/api/ruby/'
	70	+ url
	71	+ end
	72	+ end
	73	+
	74	+ version 'python' do
	75	+ self.initial_paths = %w(api/python/)
	76	+
	77	+ options[:only_patterns] += [/\Aapi\/python\//]
	78	+
	79	+ options[:fix_urls] = ->(url) do
	80	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})\\z}, 'rethinkdb.com/docs/\\1/python/'
	81	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})/(?!python/).*}, 'rethinkdb.com/docs/\\1/python/'
	82	+ url.sub! %r{rethinkdb.com/api/(?!javascript\|ruby\|python\|java)}, 'rethinkdb.com/api/python/'
	83	+ url
	84	+ end
	85	+ end
	86	+
	87	+ version 'java' do
	88	+ self.initial_paths = %w(api/java/)
	89	+
	90	+ options[:only_patterns] += [/\Aapi\/java\//]
	91	+
	92	+ options[:fix_urls] = ->(url) do
	93	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})\\z}, 'rethinkdb.com/docs/\\1/java/'
	94	+ url.sub! %r{rethinkdb.com/docs/(#{MULTILANG_DOCS.join('\|')})/(?!java/).*}, 'rethinkdb.com/docs/\\1/java/'
	95	+ url.sub! %r{rethinkdb.com/api/(?!javascript\|ruby\|python\|java)}, 'rethinkdb.com/api/java/'
	96	+ url
	97	+ end
	98	+ end
	99	+
61	100	def get_latest_version(opts)
62	101	get_latest_github_release('rethinkdb', 'rethinkdb', opts)
63	102	end
@@ -68,5 +107,6 @@ module Docs · 4 unmodified lines
68	107	return false unless super
69	108	response.body !~ /http-equiv="refresh"/i
70	109	end
	110	+
71	111	end
72	112	end