Showing the changes to lib/docs/core (2 of 6 files). Show all
Unified · Split
| @@ -3,7 +3,7 @@ require 'set' · 2 unmodified lines | |||
| 3 | module Docs | 3 | module Docs |
| 4 | class Scraper < Doc | 4 | class Scraper < Doc |
| 5 | class << self | 5 | class << self |
| 6 | attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters, :stubs | 6 | attr_accessor :base_url, :root_path, :initial_paths, :initial_urls, :options, :html_filters, :text_filters, :stubs |
| 7 | 7 | ||
| 8 | def inherited(subclass) | 8 | def inherited(subclass) |
| 9 | super | 9 | super |
| @@ -16,6 +16,7 @@ module Docs · 6 unmodified lines | |||
| 16 | subclass.base_url = base_url | 16 | subclass.base_url = base_url |
| 17 | subclass.root_path = root_path | 17 | subclass.root_path = root_path |
| 18 | subclass.initial_paths = initial_paths.dup | 18 | subclass.initial_paths = initial_paths.dup |
| 19 | subclass.initial_urls = initial_urls.dup | ||
| 19 | subclass.options = options.deep_dup | 20 | subclass.options = options.deep_dup |
| 20 | subclass.html_filters = html_filters.inheritable_copy | 21 | subclass.html_filters = html_filters.inheritable_copy |
| 21 | subclass.text_filters = text_filters.inheritable_copy | 22 | subclass.text_filters = text_filters.inheritable_copy |
| @@ -35,6 +36,7 @@ module Docs · 13 unmodified lines | |||
| 35 | include Instrumentable | 36 | include Instrumentable |
| 36 | 37 | ||
| 37 | self.initial_paths = [] | 38 | self.initial_paths = [] |
| 39 | self.initial_urls = [] | ||
| 38 | self.options = {} | 40 | self.options = {} |
| 39 | self.stubs = {} | 41 | self.stubs = {} |
| 40 | 42 | ||
| @@ -103,7 +105,7 @@ module Docs · 62 unmodified lines | |||
| 103 | end | 105 | end |
| 104 | 106 | ||
| 105 | def initial_urls | 107 | def initial_urls |
| 106 | @initial_urls ||= [root_url.to_s].concat(initial_paths.map(&method(:url_for))).freeze | 108 | @initial_urls ||= [root_url.to_s].concat(self.class.initial_urls).concat(initial_paths.map(&method(:url_for))).freeze |
| 107 | end | 109 | end |
| 108 | 110 | ||
| 109 | def pipeline | 111 | def pipeline |
| @@ -33,7 +33,11 @@ module Docs · 32 unmodified lines | |||
| 33 | raise "Error status code (#{response.code}): #{response.url}" | 33 | raise "Error status code (#{response.code}): #{response.url}" |
| 34 | end | 34 | end |
| 35 | 35 | ||
| 36 | response.success? && response.html? && base_url.contains?(response.effective_url) | 36 | response.success? && response.html? && process_url?(response.effective_url) |
| 37 | end | ||
| 38 | |||
| 39 | def process_url?(url) | ||
| 40 | base_url.contains?(url) | ||
| 37 | end | 41 | end |
| 38 | 42 | ||
| 39 | def load_capybara_selenium | 43 | def load_capybara_selenium |