History

Make Docs::FileScraper#request_all accept an array of URLs

Thibaut committed 12 years ago

Commit 5e69d6d1 parent cd6057e2 files · +48 −22Browse files

Unified · Split

2 files
  1. lib/docs/core/scrapers/file_scraper.rb +2 −2
  2. test/lib/docs/core/scrapers/file_scraper_test.rb +46 −20
lib/docs/core/scrapers/file_scraper.rb+2−2
@@ -21,8 +21,8 @@ module Docs · 20 unmodified lines
2121 Response.new read_file(file_path_for(url)), URL.parse(url)
2222 end
2323
24− def request_all(url)
25− queue = [url]
24+ def request_all(urls)
25+ queue = [urls].flatten
2626 until queue.empty?
2727 result = yield request_one(queue.shift)
2828 queue.concat(result) if result.is_a? Array
test/lib/docs/core/scrapers/file_scraper_test.rb+46−20
@@ -54,27 +54,53 @@ class FileScraperTest < MiniTest::Spec · 53 unmodified lines
5454 end
5555
5656 describe "#request_all" do
57− it "requests the given url and yields the response" do
58− stub(scraper).request_one('url') { 'response' }
59− scraper.send(:request_all, 'url') { |response| @response = response }
60− assert_equal 'response', @response
61− end
62−
63− describe "when the block returns an array" do
64− it "requests and yields the returned urls" do
65− stub(scraper).request_one('one') { 1 }
66− stub(scraper).request_one('two') { 2 }
67− stub(scraper).request_one('three') { 3 }
68− scraper.send :request_all, 'one' do |response|
69− if response == 1
70− ['two']
71− elsif response == 2
72− ['three']
73− else
74− @response = response
75− end
57+ let :urls do
58+ %w(one two)
59+ end
60+
61+ it "requests the given url" do
62+ mock(scraper).request_one('url')
63+ scraper.send(:request_all, 'url') {}
64+ end
65+
66+ it "requests the given urls" do
67+ requests = []
68+ stub(scraper).request_one { |url| requests << url; nil }
69+ scraper.send(:request_all, urls) {}
70+ assert_equal urls, requests
71+ end
72+
73+ it "yields the responses" do
74+ responses = []
75+ stub(scraper).request_one { |url| urls.index(url) }
76+ scraper.send(:request_all, urls) { |response| responses << response; nil }
77+ assert_equal (0...urls.length).to_a, responses
78+ end
79+
80+ context "when the block returns an array" do
81+ let :next_urls do
82+ %w(three four)
83+ end
84+
85+ let :all_urls do
86+ urls + %w(three four)
87+ end
88+
89+ it "requests the returned urls" do
90+ requests = []
91+ stub(scraper).request_one { |url| requests << url; url }
92+ scraper.send(:request_all, urls) { [next_urls.shift].compact }
93+ assert_equal all_urls, requests
94+ end
95+
96+ it "yields their responses" do
97+ responses = []
98+ stub(scraper).request_one { |url| all_urls.index(url) }
99+ scraper.send :request_all, urls do |response|
100+ responses << response
101+ [next_urls.shift].compact
76102 end
77− assert_equal 3, @response
103+ assert_equal (0...all_urls.length).to_a, responses
78104 end
79105 end
80106 end