History

Refactor scrapers with multiple base URLs

Thibaut Courouble committed 10 years ago

Commit 46a9ed11 parent 3eb5a0c4 files · +53 −64Browse files

Unified · Split

4 files
  1. lib/docs/core/scraper.rb +2 −4
  2. lib/docs/core/scrapers/url_scraper.rb +46 −0
  3. lib/docs/scrapers/ember.rb +2 −30
  4. lib/docs/scrapers/meteor.rb +3 −30
lib/docs/core/scraper.rb+2−4
@@ -3,7 +3,7 @@ require 'set' · 2 unmodified lines
3module Docs3module Docs
4 class Scraper < Doc4 class Scraper < Doc
5 class << self5 class << self
6 attr_accessor :base_url, :root_path, :initial_paths, :initial_urls, :options, :html_filters, :text_filters, :stubs6 attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters, :stubs
7 7
8 def inherited(subclass)8 def inherited(subclass)
9 super9 super
@@ -16,7 +16,6 @@ module Docs · 6 unmodified lines
16 subclass.base_url = base_url16 subclass.base_url = base_url
17 subclass.root_path = root_path17 subclass.root_path = root_path
18 subclass.initial_paths = initial_paths.dup18 subclass.initial_paths = initial_paths.dup
19 subclass.initial_urls = initial_urls.dup
20 subclass.options = options.deep_dup19 subclass.options = options.deep_dup
21 subclass.html_filters = html_filters.inheritable_copy20 subclass.html_filters = html_filters.inheritable_copy
22 subclass.text_filters = text_filters.inheritable_copy21 subclass.text_filters = text_filters.inheritable_copy
@@ -36,7 +35,6 @@ module Docs · 13 unmodified lines
36 include Instrumentable35 include Instrumentable
37 36
38 self.initial_paths = []37 self.initial_paths = []
39 self.initial_urls = []
40 self.options = {}38 self.options = {}
41 self.stubs = {}39 self.stubs = {}
42 40
@@ -105,7 +103,7 @@ module Docs · 62 unmodified lines
105 end103 end
106 104
107 def initial_urls105 def initial_urls
108 @initial_urls ||= [root_url.to_s].concat(self.class.initial_urls).concat(initial_paths.map(&method(:url_for))).freeze106 @initial_urls ||= [root_url.to_s].concat(initial_paths.map(&method(:url_for))).freeze
109 end107 end
110 108
111 def pipeline109 def pipeline
lib/docs/core/scrapers/url_scraper.rb+46−0
@@ -50,6 +50,52 @@ module Docs · 49 unmodified lines
50 Capybara50 Capybara
51 end51 end
52 52
53 module MultipleBaseUrls
54 def self.included(base)
55 base.extend ClassMethods
56 end
57
58 module ClassMethods
59 attr_reader :base_urls
60
61 def base_urls=(urls)
62 self.base_url = urls.first
63 @base_urls = urls
64 end
65 end
66
67 def initial_urls
68 super + self.class.base_urls[1..-1]
69 end
70
71 def base_urls
72 @base_urls ||= self.class.base_urls.map { |url| URL.parse(url) }
73 end
74
75 private
76
77 def process_url?(url)
78 base_urls.any? { |base_url| base_url.contains?(url) }
79 end
80
81 def process_response(response)
82 original_scheme = self.base_url.scheme
83 original_host = self.base_url.host
84 original_path = self.base_url.path
85
86 effective_base_url = self.base_urls.find { |base_url| base_url.contains?(response.effective_url) }
87
88 self.base_url.scheme = effective_base_url.scheme
89 self.base_url.host = effective_base_url.host
90 self.base_url.path = effective_base_url.path
91 super
92 ensure
93 self.base_url.scheme = original_scheme
94 self.base_url.host = original_host
95 self.base_url.path = original_path
96 end
97 end
98
53 module FixRedirectionsBehavior99 module FixRedirectionsBehavior
54 def self.included(base)100 def self.included(base)
55 base.extend ClassMethods101 base.extend ClassMethods
lib/docs/scrapers/ember.rb+2−30
@@ -1,16 +1,12 @@
1module Docs1module Docs
2 class Ember < UrlScraper2 class Ember < UrlScraper
3 class << self3 include MultipleBaseUrls
4 attr_accessor :guide_url
5 end
6 4
7 self.name = 'Ember.js'5 self.name = 'Ember.js'
8 self.slug = 'ember'6 self.slug = 'ember'
9 self.type = 'ember'7 self.type = 'ember'
10 self.release = '2.7.0'8 self.release = '2.7.0'
11 self.base_url = 'http://emberjs.com/api/'9 self.base_urls = ['http://emberjs.com/api/', "https://guides.emberjs.com/v#{self.release}/"]
12 self.guide_url = "https://guides.emberjs.com/v#{self.release}/"
13 self.initial_urls = [guide_url]
14 self.links = {10 self.links = {
15 home: 'http://emberjs.com/',11 home: 'http://emberjs.com/',
16 code: 'https://github.com/emberjs/ember.js'12 code: 'https://github.com/emberjs/ember.js'
@@ -39,29 +35,5 @@ module Docs · 22 unmodified lines
39 &copy; 2016 Yehuda Katz, Tom Dale and Ember.js contributors<br>35 &copy; 2016 Yehuda Katz, Tom Dale and Ember.js contributors<br>
40 Licensed under the MIT License.36 Licensed under the MIT License.
41 HTML37 HTML
42
43 def guide_url
44 @guide_url ||= URL.parse(self.class.guide_url)
45 end
46
47 private
48
49 def process_url?(url)
50 base_url.contains?(url) || guide_url.contains?(url)
51 end
52
53 def process_response(response)
54 original_scheme = @base_url.scheme
55 original_host = @base_url.host
56 original_path = @base_url.path
57 @base_url.scheme = response.effective_url.scheme
58 @base_url.host = response.effective_url.host
59 @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] || '/api/'
60 super
61 ensure
62 @base_url.scheme = original_scheme
63 @base_url.host = original_host
64 @base_url.path = original_path
65 end
66 end38 end
67end39end
lib/docs/scrapers/meteor.rb+3−30
@@ -1,8 +1,6 @@
1module Docs1module Docs
2 class Meteor < UrlScraper2 class Meteor < UrlScraper
3 class << self3 include MultipleBaseUrls
4 attr_accessor :guide_url
5 end
6 4
7 self.type = 'meteor'5 self.type = 'meteor'
8 self.root_path = 'index.html'6 self.root_path = 'index.html'
@@ -28,37 +26,12 @@ module Docs · 19 unmodified lines
28 26
29 version '1.4' do27 version '1.4' do
30 self.release = '1.4.0'28 self.release = '1.4.0'
31 self.base_url = 'https://docs.meteor.com/'29 self.base_urls = ['https://docs.meteor.com/', 'https://guide.meteor.com/']
32 self.guide_url = 'https://guide.meteor.com/'
33 self.initial_urls = [guide_url]
34 end30 end
35 31
36 version '1.3' do32 version '1.3' do
37 self.release = '1.3.5'33 self.release = '1.3.5'
38 self.base_url = "https://docs.meteor.com/v#{self.release}/"34 self.base_urls = ["https://docs.meteor.com/v#{self.release}/", 'https://guide.meteor.com/v1.3/']
39 self.guide_url = 'https://guide.meteor.com/v1.3/'
40 self.initial_urls = [guide_url]
41 end
42
43 def guide_url
44 @guide_url ||= URL.parse(self.class.guide_url)
45 end
46
47 private
48
49 def process_url?(url)
50 base_url.contains?(url) || guide_url.contains?(url)
51 end
52
53 def process_response(response)
54 original_host = @base_url.host
55 original_path = @base_url.path
56 @base_url.host = response.effective_url.host
57 @base_url.path = response.effective_url.path[/\A\/v[\d\.]+\//, 0] || '/'
58 super
59 ensure
60 @base_url.host = original_host
61 @base_url.path = original_path
62 end35 end
63 end36 end
64end37end