History

Refactor Docs::Scraper

Thibaut committed 12 years ago

Commit b92db881 parent b66d6d93 files · +115 −101Browse files

Unified · Split

3 files
  1. lib/docs/core/scraper.rb +2 −2
  2. lib/docs/subscribers/scraper_subscriber.rb +2 −0
  3. test/lib/docs/core/scraper_test.rb +111 −99
lib/docs/core/scraper.rb+2−2
@@ -45,14 +45,14 @@ module Docs · 44 unmodified lines
4545 end
4646
4747 def build_pages
48− requested_urls = Set.new initial_urls.map(&:downcase)
48+ history = Set.new initial_urls.map(&:downcase)
4949 instrument 'running.scraper', urls: initial_urls
5050
5151 request_all initial_urls do |response|
5252 next unless data = handle_response(response)
5353 yield data
5454 next unless data[:internal_urls].present?
55− next_urls = data[:internal_urls].select { |url| requested_urls.add?(url.downcase) }
55+ next_urls = data[:internal_urls].select { |url| history.add?(url.downcase) }
5656 instrument 'queued.scraper', urls: next_urls
5757 next_urls
5858 end
lib/docs/subscribers/scraper_subscriber.rb+2−0
@@ -8,6 +8,8 @@ module Docs · 7 unmodified lines
88 end
99 end
1010
11+ alias_method :running, :queued
12+
1113 def ignore_response(event)
1214 msg = "Ignore: #{format_url event.payload[:response].url}"
1315 msg << " [#{event.payload[:response].code}]" if event.payload[:response].respond_to?(:code)
test/lib/docs/core/scraper_test.rb+111−99
@@ -21,10 +21,6 @@ class DocsScraperTest < MiniTest::Spec · 20 unmodified lines
2121 Struct.new(:body, :url).new
2222 end
2323
24− let :processed_response do
25− Hash.new
26− end
27−
2824 describe ".inherited" do
2925 let :subclass do
3026 Class.new Scraper
@@ -88,33 +84,40 @@ class DocsScraperTest < MiniTest::Spec · 57 unmodified lines
8884 end
8985
9086 describe "#root_url" do
87+ let :root_url do
88+ scraper.root_url
89+ end
90+
9191 context "when #root_path? is false" do
9292 before do
9393 stub(scraper).root_path? { false }
9494 end
9595
96− it "returns a Docs::URL" do
97− assert_instance_of Docs::URL, scraper.root_url
96+ it "returns a memoized Docs::URL" do
97+ assert_instance_of Docs::URL, root_url
98+ assert_same root_url, scraper.root_url
9899 end
99100
100− it "returns the normalized base url" do
101+ it "returns the normalized .base_url" do
101102 stub(Scraper).base_url { 'http://example.com' }
102− assert_equal 'http://example.com/', scraper.root_url.to_s
103+ assert_equal 'http://example.com/', root_url.to_s
103104 end
104105 end
105106
106− context "when .root_path isn't blank" do
107+ context "when #root_path? is true" do
107108 before do
108109 stub(scraper).root_path? { true }
109110 end
110111
111− it "returns a Docs::URL" do
112− assert_instance_of Docs::URL, scraper.root_url
112+ it "returns a memoized Docs::URL" do
113+ assert_instance_of Docs::URL, root_url
114+ assert_same root_url, scraper.root_url
113115 end
114116
115− it "returns base url + root path" do
117+ it "returns .base_url + .root_path" do
116118 stub(Scraper).base_url { 'http://example.com/path/' }
117− assert_equal 'http://example.com/path/root', scraper.root_url.to_s
119+ stub(Scraper).root_path { '/root' }
120+ assert_equal 'http://example.com/path/root', root_url.to_s
118121 end
119122 end
120123 end
@@ -201,6 +204,10 @@ class DocsScraperTest < MiniTest::Spec · 80 unmodified lines
201204 Proc.new {}
202205 end
203206
207+ let :processed_response do
208+ Hash.new
209+ end
210+
204211 it "requests the #initial_urls" do
205212 mock(scraper).request_all(scraper.initial_urls)
206213 scraper.build_pages(&block)
@@ -224,11 +231,10 @@ class DocsScraperTest < MiniTest::Spec · 17 unmodified lines
224231
225232 it "yields the processed response" do
226233 scraper.build_pages { |arg| @arg = arg }
227− assert @arg
228− assert_equal processed_response, @arg
234+ assert_same processed_response, @arg
229235 end
230236
231− context "when response[:internal_urls] is empty" do
237+ context "when :internal_urls is empty" do
232238 before do
233239 processed_response[:internal_urls] = []
234240 end
@@ -244,22 +250,21 @@ class DocsScraperTest < MiniTest::Spec · 9 unmodified lines
244250 end
245251 end
246252
247− context "when response[:internal_urls] isn't empty" do
248− let :urls do
253+ context "when :internal_urls isn't empty" do
254+ let :internal_urls do
249255 ['Url']
250256 end
251257
252258 before do
253− processed_response[:internal_urls] = urls
259+ processed_response[:internal_urls] = internal_urls
254260 end
255261
256262 it "requests the urls" do
257263 scraper.build_pages(&block)
258− assert_equal urls, @next_urls
264+ assert_equal internal_urls, @next_urls
259265 end
260266
261267 it "doesn't request the same url twice irrespective of case" do
262− stub(Scraper).root_path { 'PATH' }
263268 processed_response[:internal_urls] = scraper.initial_urls.map(&:swapcase)
264269 scraper.build_pages(&block)
265270 assert_empty @next_urls
@@ -269,7 +274,7 @@ class DocsScraperTest < MiniTest::Spec · 3 unmodified lines
269274 scraper.build_pages(&block)
270275 assert scraper.last_instrumentation
271276 assert_equal 'queued.scraper', scraper.last_instrumentation[:event]
272− assert_equal urls, scraper.last_instrumentation[:payload][:urls]
277+ assert_equal internal_urls, scraper.last_instrumentation[:payload][:urls]
273278 end
274279 end
275280 end
@@ -286,34 +291,59 @@ class DocsScraperTest < MiniTest::Spec · 10 unmodified lines
286291
287292 describe "#options" do
288293 let :options do
294+ Hash.new
295+ end
296+
297+ let :result do
289298 scraper.options
290299 end
291300
301+ before do
302+ stub(Scraper).options { options }
303+ end
304+
292305 it "returns a frozen, memoized Hash" do
293− assert_instance_of Hash, options
294− assert options.frozen?
295− assert_same options, scraper.options
306+ assert_instance_of Hash, result
307+ assert result.frozen?
308+ assert_same result, scraper.options
296309 end
297310
298311 it "includes .options" do
299− stub(Scraper).options { { test: true } }
300− assert options[:test]
312+ options[:test] = true
313+ assert result[:test]
301314 end
302315
303316 it "includes #base_url" do
304− assert_equal scraper.base_url, options[:base_url]
317+ assert_equal scraper.base_url, result[:base_url]
305318 end
306319
307320 it "includes #root_url" do
308− assert_equal scraper.root_url, options[:root_url]
321+ assert_equal scraper.root_url, result[:root_url]
322+ end
323+
324+ it "includes #root_path" do
325+ assert_equal '/root', result[:root_path]
326+ end
327+
328+ it "includes #initial_paths" do
329+ assert_equal ['/initial'], result[:initial_paths]
330+ end
331+
332+ it "adds #initial_paths to :only when it is an array" do
333+ options[:only] = ['/path']
334+ assert_includes result[:only], options[:only].first
335+ assert_includes result[:only], '/initial'
309336 end
310337
311− it "includes .root_path" do
312− assert_equal '/root', options[:root_path]
338+ it "adds #initial_paths to :only when :only_patterns is an array" do
339+ options[:only_patterns] = []
340+ assert_includes result[:only], '/initial'
313341 end
314342
315− it "includes .initial_paths" do
316− assert_equal ['/initial'], options[:initial_paths]
343+ it "doesn't modify :only in place" do
344+ options[:only] = []
345+ result
346+ assert_empty options[:only]
317347 end
318348
319349 context "when #root_path? is false" do
@@ -322,42 +352,27 @@ class DocsScraperTest < MiniTest::Spec · 2 unmodified lines
322352 end
323353
324354 it "doesn't modify :skip" do
325− assert_nil options[:skip]
355+ options[:skip] = []
356+ assert_equal options[:skip], result[:skip]
326357 end
327358
328− context "and :only is an array" do
329− before do
330− stub(Scraper).options { { only: ['/path'] } }
331− end
332−
333− it "adds ['', '/']" do
334− assert_includes options[:only], '/path'
335− assert_includes options[:only], ''
336− assert_includes options[:only], '/'
337− end
338−
339− it "adds .initial_paths" do
340− assert_includes options[:only], '/initial'
341− end
342−
343− it "doesn't modify the array in place" do
344− assert_equal ['/path'], Scraper.options[:only]
345− end
359+ it "adds '' and '/' to :only when it is an array" do
360+ options[:only] = ['/path']
361+ assert_includes result[:only], options[:only].first
362+ assert_includes result[:only], ''
363+ assert_includes result[:only], '/'
346364 end
347365
348− context "and :only_patterns is an array" do
349− before do
350− stub(Scraper).options { { only_patterns: [] } }
351− end
352−
353− it "adds ['', '/'] to :only" do
354− assert_includes options[:only], ''
355− assert_includes options[:only], '/'
356− end
366+ it "adds '' and '/' to :only when :only_patterns is an array" do
367+ options[:only_patterns] = []
368+ assert_includes result[:only], ''
369+ assert_includes result[:only], '/'
370+ end
357371
358− it "adds .initial_paths to :only" do
359− assert_includes options[:only], '/initial'
360− end
372+ it "doesn't modify :only in place" do
373+ options[:only] = []
374+ result
375+ assert_empty options[:only]
361376 end
362377 end
363378
@@ -366,40 +381,33 @@ class DocsScraperTest < MiniTest::Spec · 2 unmodified lines
366381 stub(scraper).root_path? { true }
367382 end
368383
369− context "and :skip is nil" do
370− it "assigns it ['', '/']" do
371− assert_equal ['', '/'], options[:skip]
372− end
384+ it "adds '' and '/' to :skip when it is nil" do
385+ assert_includes result[:skip], ''
386+ assert_includes result[:skip], '/'
373387 end
374388
375− context "and :skip is an array" do
376− before do
377− stub(Scraper).options { { skip: ['/path'] } }
378− end
379−
380− it "adds ['', '/']" do
381− assert_includes options[:skip], '/path'
382− assert_includes options[:skip], ''
383− assert_includes options[:skip], '/'
384− end
389+ it "adds '' and '/' to :skip when it is an array" do
390+ options[:skip] = ['/path']
391+ assert_includes result[:skip], options[:skip].first
392+ assert_includes result[:skip], ''
393+ assert_includes result[:skip], '/'
394+ end
385395
386− it "doesn't modify the array in place" do
387− assert_equal ['/path'], Scraper.options[:skip]
388− end
396+ it "doesn't modify :skip in place" do
397+ options[:skip] = []
398+ result
399+ assert_empty options[:skip]
389400 end
390401
391− context "and :only is an array" do
392− it "adds .root_path" do
393− stub(Scraper).options { { only: [] } }
394− assert_includes options[:only], '/root'
395− end
402+ it "adds #root_path to :only when it is an array" do
403+ options[:only] = ['/path']
404+ assert_includes result[:only], options[:only].first
405+ assert_includes result[:only], '/root'
396406 end
397407
398− context "and :only_patterns is an array" do
399− it "adds .root_path to :only" do
400− stub(Scraper).options { { only_patterns: [] } }
401− assert_includes options[:only], '/root'
402− end
408+ it "adds #root_path to :only when :only_patterns is an array" do
409+ options[:only_patterns] = []
410+ assert_includes result[:only], '/root'
403411 end
404412 end
405413 end
@@ -484,18 +492,22 @@ class DocsScraperTest < MiniTest::Spec · 78 unmodified lines
484492 end
485493
486494 describe "#pipeline" do
487− it "returns an HTML::Pipeline with .filters" do
488− stub(Scraper).filters { [1] }
489− assert_instance_of ::HTML::Pipeline, scraper.pipeline
490− assert_equal Scraper.filters, scraper.pipeline.filters
495+ let :pipeline do
496+ scraper.pipeline
491497 end
492498
493− it "is memoized" do
494− assert_same scraper.pipeline, scraper.pipeline
499+ it "returns a memoized HTML::Pipeline" do
500+ assert_instance_of ::HTML::Pipeline, pipeline
501+ assert_same pipeline, scraper.pipeline
502+ end
503+
504+ it "returns a pipeline with the filters stored in .filters" do
505+ stub(Scraper).filters { [1] }
506+ assert_equal Scraper.filters, pipeline.filters
495507 end
496508
497− it "assigns Docs as the pipeline's instrumentation service" do
498− assert_equal Docs, scraper.pipeline.instrumentation_service
509+ it "returns a pipeline with Docs as instrumentation service" do
510+ assert_equal Docs, pipeline.instrumentation_service
499511 end
500512 end
501513 end