Make Docs::FileScraper#request_all accept an array of URLs

Thibaut committed 12 years ago

Commit 5e69d6d1 parent cd6057e2 files · +48 −22Browse files

Unified · Split

2 files

lib/docs/core/scrapers/file_scraper.rb +2 −2
test/lib/docs/core/scrapers/file_scraper_test.rb +46 −20

lib/docs/core/scrapers/file_scraper.rb+2−2

@@ -21,8 +21,8 @@ module Docs · 20 unmodified lines
21	Response.new read_file(file_path_for(url)), URL.parse(url)	21	Response.new read_file(file_path_for(url)), URL.parse(url)
22	end	22	end
23		23
24	def request_all(url)	24	def request_all(urls)
25	queue = [url]	25	queue = [urls].flatten
26	until queue.empty?	26	until queue.empty?
27	result = yield request_one(queue.shift)	27	result = yield request_one(queue.shift)
28	queue.concat(result) if result.is_a? Array	28	queue.concat(result) if result.is_a? Array

test/lib/docs/core/scrapers/file_scraper_test.rb+46−20

@@ -54,27 +54,53 @@ class FileScraperTest < MiniTest::Spec · 53 unmodified lines
54	end	54	end
55		55
56	describe "#request_all" do	56	describe "#request_all" do
57	it "requests the given url and yields the response" do	57	let :urls do
58	stub(scraper).request_one('url') { 'response' }	58	%w(one two)
59	scraper.send(:request_all, 'url') { \|response\| @response = response }	59	end
60	assert_equal 'response', @response	60
61	end	61	it "requests the given url" do
62		62	mock(scraper).request_one('url')
63	describe "when the block returns an array" do	63	scraper.send(:request_all, 'url') {}
64	it "requests and yields the returned urls" do	64	end
65	stub(scraper).request_one('one') { 1 }	65
66	stub(scraper).request_one('two') { 2 }	66	it "requests the given urls" do
67	stub(scraper).request_one('three') { 3 }	67	requests = []
68	scraper.send :request_all, 'one' do \|response\|	68	stub(scraper).request_one { \|url\| requests << url; nil }
69	if response == 1	69	scraper.send(:request_all, urls) {}
70	['two']	70	assert_equal urls, requests
71	elsif response == 2	71	end
72	['three']	72
73	else	73	it "yields the responses" do
74	@response = response	74	responses = []
75	end	75	stub(scraper).request_one { \|url\| urls.index(url) }
		76	scraper.send(:request_all, urls) { \|response\| responses << response; nil }
		77	assert_equal (0...urls.length).to_a, responses
		78	end
		79
		80	context "when the block returns an array" do
		81	let :next_urls do
		82	%w(three four)
		83	end
		84
		85	let :all_urls do
		86	urls + %w(three four)
		87	end
		88
		89	it "requests the returned urls" do
		90	requests = []
		91	stub(scraper).request_one { \|url\| requests << url; url }
		92	scraper.send(:request_all, urls) { [next_urls.shift].compact }
		93	assert_equal all_urls, requests
		94	end
		95
		96	it "yields their responses" do
		97	responses = []
		98	stub(scraper).request_one { \|url\| all_urls.index(url) }
		99	scraper.send :request_all, urls do \|response\|
		100	responses << response
		101	[next_urls.shift].compact
76	end	102	end
77	assert_equal 3, @response	103	assert_equal (0...all_urls.length).to_a, responses
78	end	104	end
79	end	105	end
80	end	106	end