History

Refactor FixRedirectionsBehavior

Thibaut Courouble committed 10 years ago

Commit b8658d91 parent 9c53ba14 files · +13 −15Browse files

Unified · Split

4 files
  1. lib/docs/core/scraper.rb +9 −0
  2. lib/docs/core/scrapers/url_scraper.rb +0 −13
  3. lib/docs/scrapers/mdn/dom.rb +2 −1
  4. lib/docs/scrapers/mdn/svg.rb +2 −1
lib/docs/core/scraper.rb+9−0
@@ -154,6 +154,15 @@ module Docs · 153 unmodified lines
154 Parser.new(string).html154 Parser.new(string).html
155 end155 end
156 156
157 def with_filters(*filters)
158 stack = FilterStack.new
159 stack.push(*filters)
160 pipeline.instance_variable_set :@filters, stack.to_a.freeze
161 yield
162 ensure
163 @pipeline = nil
164 end
165
157 module StubRootPage166 module StubRootPage
158 private167 private
159 168
lib/docs/core/scrapers/url_scraper.rb+0−13
@@ -42,11 +42,9 @@ module Docs · 41 unmodified lines
42 end42 end
43 43
44 module ClassMethods44 module ClassMethods
45 attr_accessor :fix_redirections
46 attr_reader :redirections45 attr_reader :redirections
47 46
48 def store_pages(store)47 def store_pages(store)
49 return super unless fix_redirections
50 instrument 'info.doc', msg: 'Fetching redirections...'48 instrument 'info.doc', msg: 'Fetching redirections...'
51 with_redirections do49 with_redirections do
52 instrument 'info.doc', msg: 'Building pages...'50 instrument 'info.doc', msg: 'Building pages...'
@@ -84,17 +82,6 @@ module Docs · 31 unmodified lines
84 def additional_options82 def additional_options
85 { redirections: self.class.redirections }83 { redirections: self.class.redirections }
86 end84 end
87
88 def with_filters(*filters)
89 stack = FilterStack.new
90 stack.push(*filters)
91 pipeline.instance_variable_set :@filters, stack.to_a.freeze
92 yield
93 ensure
94 @pipeline = nil
95 end
96 end85 end
97
98 include FixRedirectionsBehavior
99 end86 end
100end87end
lib/docs/scrapers/mdn/dom.rb+2−1
@@ -1,8 +1,9 @@
1module Docs1module Docs
2 class Dom < Mdn2 class Dom < Mdn
3 include FixRedirectionsBehavior
4
3 self.name = 'DOM'5 self.name = 'DOM'
4 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'6 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/API'
5 self.fix_redirections = true
6 7
7 html_filters.push 'dom/clean_html', 'dom/entries', 'title'8 html_filters.push 'dom/clean_html', 'dom/entries', 'title'
8 9
lib/docs/scrapers/mdn/svg.rb+2−1
@@ -1,8 +1,9 @@
1module Docs1module Docs
2 class Svg < Mdn2 class Svg < Mdn
3 include FixRedirectionsBehavior
4
3 self.name = 'SVG'5 self.name = 'SVG'
4 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/SVG'6 self.base_url = 'https://developer.mozilla.org/en-US/docs/Web/SVG'
5 self.fix_redirections = true
6 7
7 html_filters.push 'svg/clean_html', 'svg/entries', 'title'8 html_filters.push 'svg/clean_html', 'svg/entries', 'title'
8 9