History

Move code in scraper.rb

Thibaut committed 13 years ago

Commit 5c477521 parent d964fd41 file · +29 −24Browse files

Unified · Split

lib/docs/core/scraper.rb+29−24
@@ -35,22 +35,6 @@ module Docs · 34 unmodified lines
35 html_filters.push 'container', 'clean_html', 'normalize_urls', 'internal_urls', 'normalize_paths'35 html_filters.push 'container', 'clean_html', 'normalize_urls', 'internal_urls', 'normalize_paths'
36 text_filters.push 'inner_html', 'clean_text', 'attribution'36 text_filters.push 'inner_html', 'clean_text', 'attribution'
37 37
38 def base_url
39 @base_url ||= URL.parse self.class.base_url
40 end
41
42 def root_url
43 @root_url ||= root_path? ? URL.parse(File.join(base_url.to_s, root_path)) : base_url.normalize
44 end
45
46 def root_path
47 self.class.root_path
48 end
49
50 def root_path?
51 root_path.present? && root_path != '/'
52 end
53
54 def build_page(path)38 def build_page(path)
55 response = request_one url_for(path)39 response = request_one url_for(path)
56 result = handle_response(response)40 result = handle_response(response)
@@ -72,20 +56,41 @@ module Docs · 15 unmodified lines
72 end56 end
73 end57 end
74 58
59 def base_url
60 @base_url ||= URL.parse self.class.base_url
61 end
62
63 def root_url
64 @root_url ||= root_path? ? URL.parse(File.join(base_url.to_s, root_path)) : base_url.normalize
65 end
66
67 def root_path
68 self.class.root_path
69 end
70
71 def root_path?
72 root_path.present? && root_path != '/'
73 end
74
75 def pipeline
76 @pipeline ||= ::HTML::Pipeline.new(self.class.filters).tap do |pipeline|
77 pipeline.instrumentation_service = Docs
78 end
79 end
80
75 def options81 def options
76 @options ||= self.class.options.deep_dup.tap do |options|82 @options ||= self.class.options.deep_dup.tap do |options|
77 options.merge! base_url: base_url, root_path: root_path, root_url: root_url83 options.merge! base_url: base_url, root_url: root_url, root_path: root_path
78 (options[:skip] ||= []).concat ['', '/'] if root_path?84
85 if root_path?
86 (options[:skip] ||= []).concat ['', '/']
87 end
88
79 if options[:only] || options[:only_patterns]89 if options[:only] || options[:only_patterns]
80 (options[:only] ||= []).concat root_path? ? [root_path] : ['', '/']90 (options[:only] ||= []).concat root_path? ? [root_path] : ['', '/']
81 end91 end
82 options.freeze
83 end
84 end
85 92
86 def pipeline93 options.freeze
87 @pipeline ||= ::HTML::Pipeline.new(self.class.filters).tap do |pipeline|
88 pipeline.instrumentation_service = Docs
89 end94 end
90 end95 end
91 96