History

Move code in scraper.rb

Thibaut committed 13 years ago

Commit 5c477521 parent d964fd41 file · +29 −24Browse files

Unified · Split

lib/docs/core/scraper.rb+29−24
@@ -35,22 +35,6 @@ module Docs · 34 unmodified lines
3535 html_filters.push 'container', 'clean_html', 'normalize_urls', 'internal_urls', 'normalize_paths'
3636 text_filters.push 'inner_html', 'clean_text', 'attribution'
3737
38− def base_url
39− @base_url ||= URL.parse self.class.base_url
40− end
41−
42− def root_url
43− @root_url ||= root_path? ? URL.parse(File.join(base_url.to_s, root_path)) : base_url.normalize
44− end
45−
46− def root_path
47− self.class.root_path
48− end
49−
50− def root_path?
51− root_path.present? && root_path != '/'
52− end
53−
5438 def build_page(path)
5539 response = request_one url_for(path)
5640 result = handle_response(response)
@@ -72,20 +56,41 @@ module Docs · 15 unmodified lines
7256 end
7357 end
7458
59+ def base_url
60+ @base_url ||= URL.parse self.class.base_url
61+ end
62+
63+ def root_url
64+ @root_url ||= root_path? ? URL.parse(File.join(base_url.to_s, root_path)) : base_url.normalize
65+ end
66+
67+ def root_path
68+ self.class.root_path
69+ end
70+
71+ def root_path?
72+ root_path.present? && root_path != '/'
73+ end
74+
75+ def pipeline
76+ @pipeline ||= ::HTML::Pipeline.new(self.class.filters).tap do |pipeline|
77+ pipeline.instrumentation_service = Docs
78+ end
79+ end
80+
7581 def options
7682 @options ||= self.class.options.deep_dup.tap do |options|
77− options.merge! base_url: base_url, root_path: root_path, root_url: root_url
78− (options[:skip] ||= []).concat ['', '/'] if root_path?
83+ options.merge! base_url: base_url, root_url: root_url, root_path: root_path
84+
85+ if root_path?
86+ (options[:skip] ||= []).concat ['', '/']
87+ end
88+
7989 if options[:only] || options[:only_patterns]
8090 (options[:only] ||= []).concat root_path? ? [root_path] : ['', '/']
8191 end
82− options.freeze
83− end
84− end
8592
86− def pipeline
87− @pipeline ||= ::HTML::Pipeline.new(self.class.filters).tap do |pipeline|
88− pipeline.instrumentation_service = Docs
93+ options.freeze
8994 end
9095 end
9196