Refactor scrapers with multiple base URLs

Thibaut Courouble committed 10 years ago

Commit 46a9ed11 parent 3eb5a0c4 files · +53 −64Browse files

Unified · Split

4 files

@@ -3,7 +3,7 @@ require 'set' · 2 unmodified lines
3	3	module Docs
4	4	class Scraper < Doc
5	5	class << self
6		− attr_accessor :base_url, :root_path, :initial_paths, :initial_urls, :options, :html_filters, :text_filters, :stubs
	6	+ attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters, :stubs
7	7
8	8	def inherited(subclass)
9	9	super
@@ -16,7 +16,6 @@ module Docs · 6 unmodified lines
16	16	subclass.base_url = base_url
17	17	subclass.root_path = root_path
18	18	subclass.initial_paths = initial_paths.dup
19		− subclass.initial_urls = initial_urls.dup
20	19	subclass.options = options.deep_dup
21	20	subclass.html_filters = html_filters.inheritable_copy
22	21	subclass.text_filters = text_filters.inheritable_copy
@@ -36,7 +35,6 @@ module Docs · 13 unmodified lines
36	35	include Instrumentable
37	36
38	37	self.initial_paths = []
39		− self.initial_urls = []
40	38	self.options = {}
41	39	self.stubs = {}
42	40
@@ -105,7 +103,7 @@ module Docs · 62 unmodified lines
105	103	end
106	104
107	105	def initial_urls
108		− @initial_urls \|\|= [root_url.to_s].concat(self.class.initial_urls).concat(initial_paths.map(&method(:url_for))).freeze
	106	+ @initial_urls \|\|= [root_url.to_s].concat(initial_paths.map(&method(:url_for))).freeze
109	107	end
110	108
111	109	def pipeline

@@ -50,6 +50,52 @@ module Docs · 49 unmodified lines
50	50	Capybara
51	51	end
52	52
	53	+ module MultipleBaseUrls
	54	+ def self.included(base)
	55	+ base.extend ClassMethods
	56	+ end
	57	+
	58	+ module ClassMethods
	59	+ attr_reader :base_urls
	60	+
	61	+ def base_urls=(urls)
	62	+ self.base_url = urls.first
	63	+ @base_urls = urls
	64	+ end
	65	+ end
	66	+
	67	+ def initial_urls
	68	+ super + self.class.base_urls[1..-1]
	69	+ end
	70	+
	71	+ def base_urls
	72	+ @base_urls \|\|= self.class.base_urls.map { \|url\| URL.parse(url) }
	73	+ end
	74	+
	75	+ private
	76	+
	77	+ def process_url?(url)
	78	+ base_urls.any? { \|base_url\| base_url.contains?(url) }
	79	+ end
	80	+
	81	+ def process_response(response)
	82	+ original_scheme = self.base_url.scheme
	83	+ original_host = self.base_url.host
	84	+ original_path = self.base_url.path
	85	+
	86	+ effective_base_url = self.base_urls.find { \|base_url\| base_url.contains?(response.effective_url) }
	87	+
	88	+ self.base_url.scheme = effective_base_url.scheme
	89	+ self.base_url.host = effective_base_url.host
	90	+ self.base_url.path = effective_base_url.path
	91	+ super
	92	+ ensure
	93	+ self.base_url.scheme = original_scheme
	94	+ self.base_url.host = original_host
	95	+ self.base_url.path = original_path
	96	+ end
	97	+ end
	98	+
53	99	module FixRedirectionsBehavior
54	100	def self.included(base)
55	101	base.extend ClassMethods

@@ -1,16 +1,12 @@
1	1	module Docs
2	2	class Ember < UrlScraper
3		− class << self
4		− attr_accessor :guide_url
5		− end
	3	+ include MultipleBaseUrls
6	4
7	5	self.name = 'Ember.js'
8	6	self.slug = 'ember'
9	7	self.type = 'ember'
10	8	self.release = '2.7.0'
11		− self.base_url = 'http://emberjs.com/api/'
12		− self.guide_url = "https://guides.emberjs.com/v#{self.release}/"
13		− self.initial_urls = [guide_url]
	9	+ self.base_urls = ['http://emberjs.com/api/', "https://guides.emberjs.com/v#{self.release}/"]
14	10	self.links = {
15	11	home: 'http://emberjs.com/',
16	12	code: 'https://github.com/emberjs/ember.js'
@@ -39,29 +35,5 @@ module Docs · 22 unmodified lines
39	35	© 2016 Yehuda Katz, Tom Dale and Ember.js contributors<br>
40	36	Licensed under the MIT License.
41	37	HTML
42		−
43		− def guide_url
44		− @guide_url \|\|= URL.parse(self.class.guide_url)
45		− end
46		−
47		− private
48		−
49		− def process_url?(url)
50		− base_url.contains?(url) \|\| guide_url.contains?(url)
51		− end
52		−
53		− def process_response(response)
54		− original_scheme = @base_url.scheme
55		− original_host = @base_url.host
56		− original_path = @base_url.path
57		− @base_url.scheme = response.effective_url.scheme
58		− @base_url.host = response.effective_url.host
59		− @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] \|\| '/api/'
60		− super
61		− ensure
62		− @base_url.scheme = original_scheme
63		− @base_url.host = original_host
64		− @base_url.path = original_path
65		− end
66	38	end
67	39	end

@@ -1,8 +1,6 @@
1	1	module Docs
2	2	class Meteor < UrlScraper
3		− class << self
4		− attr_accessor :guide_url
5		− end
	3	+ include MultipleBaseUrls
6	4
7	5	self.type = 'meteor'
8	6	self.root_path = 'index.html'
@@ -28,37 +26,12 @@ module Docs · 19 unmodified lines
28	26
29	27	version '1.4' do
30	28	self.release = '1.4.0'
31		− self.base_url = 'https://docs.meteor.com/'
32		− self.guide_url = 'https://guide.meteor.com/'
33		− self.initial_urls = [guide_url]
	29	+ self.base_urls = ['https://docs.meteor.com/', 'https://guide.meteor.com/']
34	30	end
35	31
36	32	version '1.3' do
37	33	self.release = '1.3.5'
38		− self.base_url = "https://docs.meteor.com/v#{self.release}/"
39		− self.guide_url = 'https://guide.meteor.com/v1.3/'
40		− self.initial_urls = [guide_url]
41		− end
42		−
43		− def guide_url
44		− @guide_url \|\|= URL.parse(self.class.guide_url)
45		− end
46		−
47		− private
48		−
49		− def process_url?(url)
50		− base_url.contains?(url) \|\| guide_url.contains?(url)
51		− end
52		−
53		− def process_response(response)
54		− original_host = @base_url.host
55		− original_path = @base_url.path
56		− @base_url.host = response.effective_url.host
57		− @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] \|\| '/'
58		− super
59		− ensure
60		− @base_url.host = original_host
61		− @base_url.path = original_path
	34	+ self.base_urls = ["https://docs.meteor.com/v#{self.release}/", 'https://guide.meteor.com/v1.3/']
62	35	end
63	36	end
64	37	end