Simplify file scraper setup; scrape files in the "docs/[slug]" directory

Thibaut Courouble committed 7 years ago

Commit bf003661 parent 91753ce22 files · +124 −104Browse files

Unified · Split

22 files

.gitignore +1 −0
lib/docs.rb +1 −0
lib/docs/core/doc.rb +6 −0
lib/docs/core/scrapers/file_scraper.rb +18 −7
lib/docs/scrapers/c.rb +0 −1
lib/docs/scrapers/cpp.rb +0 −1
lib/docs/scrapers/dart.rb +0 −2
lib/docs/scrapers/django.rb +0 −6
lib/docs/scrapers/erlang.rb +0 −4
lib/docs/scrapers/gnu/gcc.rb +0 −8
lib/docs/scrapers/gnu/gnu_fortran.rb +0 −4
lib/docs/scrapers/nokogiri2.rb +0 −1
lib/docs/scrapers/numpy.rb +0 −1
lib/docs/scrapers/openjdk.rb +2 −3
lib/docs/scrapers/perl.rb +0 −1
lib/docs/scrapers/php.rb +2 −3
lib/docs/scrapers/python.rb +4 −8
lib/docs/scrapers/rdoc/minitest.rb +1 −1
lib/docs/scrapers/rdoc/rails.rb +0 −1
lib/docs/scrapers/rdoc/ruby.rb +0 −4
lib/docs/scrapers/sqlite.rb +0 −1
test/lib/docs/core/scrapers/file_scraper_test.rb +89 −47

@@ -8,3 +8,4 @@ public/fonts · 7 unmodified lines
8	8	public/docs/*/
9	9	!public/docs/docs.json
10	10	!public/docs/**/index.json
	11	+/docs/

lib/docs.rb+1−0

@@ -29,6 +29,7 @@ module Docs · 28 unmodified lines
29	29	self.rescue_errors = false
30	30
31	31	class DocNotFound < NameError; end
	32	+ class SetupError < StandardError; end
32	33
33	34	def self.all
34	35	Dir["#{root_path}/docs/scrapers/*/.rb"].

lib/docs/core/doc.rb+6−0

@@ -95,6 +95,9 @@ module Docs · 94 unmodified lines
95	95	false
96	96	end
97	97	end
	98	+ rescue Docs::SetupError => error
	99	+ puts "ERROR: #{error.message}"
	100	+ false
98	101	end
99	102
100	103	def store_pages(store)
@@ -118,6 +121,9 @@ module Docs · 17 unmodified lines
118	121	false
119	122	end
120	123	end
	124	+ rescue Docs::SetupError => error
	125	+ puts "ERROR: #{error.message}"
	126	+ false
121	127	end
122	128
123	129	private

lib/docs/core/scrapers/file_scraper.rb+18−7

@@ -1,14 +1,13 @@
1	1	module Docs
2	2	class FileScraper < Scraper
	3	+ SOURCE_DIRECTORY = File.expand_path '../../../../../docs', __FILE__
	4	+
3	5	Response = Struct.new :body, :url
4	6
5	7	class << self
6		− attr_accessor :dir
7		−
8	8	def inherited(subclass)
9	9	super
10	10	subclass.base_url = base_url
11		− subclass.dir = dir
12	11	end
13	12	end
14	13
@@ -16,13 +15,25 @@ module Docs · 1 unmodified line
16	15
17	16	html_filters.push 'clean_local_urls'
18	17
	18	+ def source_directory
	19	+ @source_directory \|\|= File.join(SOURCE_DIRECTORY, self.class.path)
	20	+ end
	21	+
19	22	private
20	23
	24	+ def assert_source_directory_exists
	25	+ unless Dir.exists?(source_directory)
	26	+ raise SetupError, "The #{self.class.name} scraper requires the original documentation files to be stored in the \"#{source_directory}\" directory."
	27	+ end
	28	+ end
	29	+
21	30	def request_one(url)
22		− Response.new read_file(file_path_for(url)), URL.parse(url)
	31	+ assert_source_directory_exists
	32	+ Response.new read_file(url_to_path(url)), URL.parse(url)
23	33	end
24	34
25	35	def request_all(urls)
	36	+ assert_source_directory_exists
26	37	queue = [urls].flatten
27	38	until queue.empty?
28	39	result = yield request_one(queue.shift)
@@ -34,12 +45,12 @@ module Docs · 5 unmodified lines
34	45	response.body.present?
35	46	end
36	47
37		− def file_path_for(url)
38		− File.join self.class.dir, url.remove(base_url.to_s)
	48	+ def url_to_path(url)
	49	+ url.remove(base_url.to_s)
39	50	end
40	51
41	52	def read_file(path)
42		− File.read(path)
	53	+ File.read(File.join(source_directory, path))
43	54	rescue
44	55	instrument 'warn.doc', msg: "Failed to open file: #{path}"
45	56	nil

lib/docs/scrapers/c.rb+0−1

@@ -1,7 +1,6 @@
1	1	module Docs
2	2	class C < FileScraper
3	3	self.type = 'c'
4		− self.dir = '/Users/Thibaut/DevDocs/Docs/c'
5	4	self.base_url = 'http://en.cppreference.com/w/c/'
6	5	self.root_path = 'header.html'
7	6

lib/docs/scrapers/cpp.rb+0−1

@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3	3	self.name = 'C++'
4	4	self.slug = 'cpp'
5	5	self.type = 'c'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/cpp'
7	6	self.base_url = 'http://en.cppreference.com/w/cpp/'
8	7	self.root_path = 'header.html'
9	8

lib/docs/scrapers/dart.rb+0−2

@@ -24,13 +24,11 @@ module Docs · 23 unmodified lines
24	24
25	25	version '2' do
26	26	self.release = '2.0.0'
27		− self.dir = '/Users/Thibaut/DevDocs/Docs/Dart2'
28	27	self.base_url = "https://api.dartlang.org/stable/#{release}/"
29	28	end
30	29
31	30	version '1' do
32	31	self.release = '1.24.3'
33		− self.dir = '/Users/Thibaut/DevDocs/Docs/Dart1'
34	32	self.base_url = "https://api.dartlang.org/stable/#{release}/"
35	33	end
36	34	end

lib/docs/scrapers/django.rb+0−6

@@ -36,37 +36,31 @@ module Docs · 35 unmodified lines
36	36
37	37	version '2.1' do
38	38	self.release = '2.1.0'
39		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django21'
40	39	self.base_url = 'https://docs.djangoproject.com/en/2.1/'
41	40	end
42	41
43	42	version '2.0' do
44	43	self.release = '2.0.7'
45		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django20'
46	44	self.base_url = 'https://docs.djangoproject.com/en/2.0/'
47	45	end
48	46
49	47	version '1.11' do
50	48	self.release = '1.11.9'
51		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django111'
52	49	self.base_url = 'https://docs.djangoproject.com/en/1.11/'
53	50	end
54	51
55	52	version '1.10' do
56	53	self.release = '1.10.8'
57		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django110'
58	54	self.base_url = 'https://docs.djangoproject.com/en/1.10/'
59	55	end
60	56
61	57	version '1.9' do
62	58	self.release = '1.9.13'
63		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django19'
64	59	self.base_url = 'https://docs.djangoproject.com/en/1.9/'
65	60	end
66	61
67	62	version '1.8' do
68	63	self.release = '1.8.18'
69		− self.dir = '/Users/Thibaut/DevDocs/Docs/Django18'
70	64	self.base_url = 'https://docs.djangoproject.com/en/1.8/'
71	65	end
72	66	end

lib/docs/scrapers/erlang.rb+0−4

@@ -42,22 +42,18 @@ module Docs · 41 unmodified lines
42	42
43	43	version '21' do
44	44	self.release = '21.0'
45		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang21'
46	45	end
47	46
48	47	version '20' do
49	48	self.release = '20.3'
50		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang20'
51	49	end
52	50
53	51	version '19' do
54	52	self.release = '19.3'
55		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang19'
56	53	end
57	54
58	55	version '18' do
59	56	self.release = '18.3'
60		− self.dir = '/Users/Thibaut/DevDocs/Docs/Erlang18'
61	57	end
62	58	end
63	59	end

lib/docs/scrapers/gnu/gcc.rb+0−8

@@ -48,13 +48,11 @@ module Docs · 47 unmodified lines
48	48
49	49	version '7' do
50	50	self.release = '7.3.0'
51		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc7'
52	51	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
53	52	end
54	53
55	54	version '7 CPP' do
56	55	self.release = '7.3.0'
57		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp7'
58	56	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
59	57
60	58	options[:replace_paths] = CPP_PATHS
@@ -62,7 +60,6 @@ module Docs · 1 unmodified line
62	60
63	61	version '6' do
64	62	self.release = '6.4.0'
65		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc6'
66	63	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
67	64
68	65	options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -70,7 +67,6 @@ module Docs · 1 unmodified line
70	67
71	68	version '6 CPP' do
72	69	self.release = '6.4.0'
73		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp6'
74	70	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
75	71
76	72	options[:replace_paths] = CPP_PATHS
@@ -78,7 +74,6 @@ module Docs · 1 unmodified line
78	74
79	75	version '5' do
80	76	self.release = '5.4.0'
81		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc5'
82	77	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
83	78
84	79	options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -86,7 +81,6 @@ module Docs · 1 unmodified line
86	81
87	82	version '5 CPP' do
88	83	self.release = '5.4.0'
89		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp5'
90	84	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
91	85
92	86	options[:replace_paths] = CPP_PATHS
@@ -94,7 +88,6 @@ module Docs · 1 unmodified line
94	88
95	89	version '4' do
96	90	self.release = '4.9.3'
97		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcc4'
98	91	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gcc/"
99	92
100	93	options[:root_title] = 'Using the GNU Compiler Collection (GCC)'
@@ -102,7 +95,6 @@ module Docs · 1 unmodified line
102	95
103	96	version '4 CPP' do
104	97	self.release = '4.9.3'
105		− self.dir = '/Users/Thibaut/DevDocs/Docs/gcpp4'
106	98	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/cpp/"
107	99
108	100	options[:replace_paths] = CPP_PATHS

lib/docs/scrapers/gnu/gnu_fortran.rb+0−4

@@ -8,25 +8,21 @@ module Docs · 7 unmodified lines
8	8
9	9	version '7' do
10	10	self.release = '7.3.0'
11		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran7'
12	11	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
13	12	end
14	13
15	14	version '6' do
16	15	self.release = '6.4.0'
17		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran6'
18	16	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
19	17	end
20	18
21	19	version '5' do
22	20	self.release = '5.4.0'
23		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran5'
24	21	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
25	22	end
26	23
27	24	version '4' do
28	25	self.release = '4.9.3'
29		− self.dir = '/Users/Thibaut/DevDocs/Docs/gfortran4'
30	26	self.base_url = "https://gcc.gnu.org/onlinedocs/gcc-#{release}/gfortran/"
31	27	end
32	28	end

lib/docs/scrapers/nokogiri2.rb+0−1

@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3	3	self.name = 'Nokogiri'
4	4	self.slug = 'nokogiri'
5	5	self.release = '1.8.1'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Nokogiri'
7	6
8	7	html_filters.replace 'rdoc/entries', 'nokogiri2/entries'
9	8

lib/docs/scrapers/numpy.rb+0−1

@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
2	2	class Numpy < FileScraper
3	3	self.name = 'NumPy'
4	4	self.type = 'sphinx'
5		− self.dir = '/Users/Thibaut/DevDocs/Docs/numpy/reference/'
6	5	self.root_path = 'index.html'
7	6	self.links = {
8	7	home: 'http://www.numpy.org/',

lib/docs/scrapers/openjdk.rb+2−3

@@ -1,11 +1,10 @@
1	1	module Docs
2	2	class Openjdk < FileScraper
	3	+ # Downloaded from packages.debian.org/sid/openjdk-8-doc
	4	+ # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
3	5	self.name = 'OpenJDK'
4	6	self.type = 'openjdk'
5	7	self.root_path = 'overview-summary.html'
6		− # Downloaded from packages.debian.org/sid/openjdk-8-doc
7		− # Extracting subdirectory /usr/share/doc/openjdk-8-jre-headless/api
8		− self.dir = '/Users/Thibaut/DevDocs/Docs/OpenJDK'
9	8
10	9	html_filters.insert_after 'internal_urls', 'openjdk/clean_urls'
11	10	html_filters.push 'openjdk/entries', 'openjdk/clean_html'

lib/docs/scrapers/perl.rb+0−1

@@ -2,7 +2,6 @@ module Docs · 1 unmodified line
2	2	class Perl < FileScraper
3	3	self.name = 'Perl'
4	4	self.type = 'perl'
5		− self.dir = '/Users/Thibaut/DevDocs/Docs/Perl'
6	5	self.root_path = 'index.html'
7	6	self.links = {
8	7	home: 'https://www.perl.org/'

lib/docs/scrapers/php.rb+2−3

@@ -1,5 +1,7 @@
1	1	module Docs
2	2	class Php < FileScraper
	3	+ # Downloaded from php.net/download-docs.php
	4	+
3	5	include FixInternalUrlsBehavior
4	6
5	7	self.name = 'PHP'
@@ -23,9 +25,6 @@ module Docs · 17 unmodified lines
23	25	code: 'https://git.php.net/?p=php-src.git;a=summary'
24	26	}
25	27
26		− # Downloaded from php.net/download-docs.php
27		− self.dir = '/Users/Thibaut/DevDocs/Docs/PHP'
28		−
29	28	html_filters.push 'php/internal_urls', 'php/entries', 'php/clean_html', 'title'
30	29	text_filters.push 'php/fix_urls'
31	30

lib/docs/scrapers/python.rb+4−8

@@ -23,33 +23,29 @@ module Docs · 22 unmodified lines
23	23	Licensed under the PSF License.
24	24	HTML
25	25
26		− version '3.7' do
	26	+ version '3.7' do # docs.python.org/3.7/download.html
27	27	self.release = '3.7.0'
28		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python37' # docs.python.org/3.7/download.html
29	28	self.base_url = 'https://docs.python.org/3.7/'
30	29
31	30	html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
32	31	end
33	32
34		− version '3.6' do
	33	+ version '3.6' do # docs.python.org/3.6/download.html
35	34	self.release = '3.6.6'
36		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python36' # docs.python.org/3.6/download.html
37	35	self.base_url = 'https://docs.python.org/3.6/'
38	36
39	37	html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
40	38	end
41	39
42		− version '3.5' do
	40	+ version '3.5' do # docs.python.org/3.5/download.html
43	41	self.release = '3.5.3'
44		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python35' # docs.python.org/3.5/download.html
45	42	self.base_url = 'https://docs.python.org/3.5/'
46	43
47	44	html_filters.push 'python/entries_v3', 'sphinx/clean_html', 'python/clean_html'
48	45	end
49	46
50		− version '2.7' do
	47	+ version '2.7' do # docs.python.org/2.7/download.html
51	48	self.release = '2.7.13'
52		− self.dir = '/Users/Thibaut/DevDocs/Docs/Python27' # docs.python.org/2.7/download.html
53	49	self.base_url = 'https://docs.python.org/2.7/'
54	50
55	51	html_filters.push 'python/entries_v2', 'sphinx/clean_html', 'python/clean_html'

lib/docs/scrapers/rdoc/minitest.rb+1−1

@@ -1,9 +1,9 @@
1	1	module Docs
2	2	class Minitest < Rdoc
	3	+ # Run "rake docs" in the gem directory
3	4	self.name = 'Ruby / Minitest'
4	5	self.slug = 'minitest'
5	6	self.release = '5.10.3'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Minitest' # rake docs
7	7	self.links = {
8	8	code: 'https://github.com/seattlerb/minitest'
9	9	}

lib/docs/scrapers/rdoc/rails.rb+0−1

@@ -4,7 +4,6 @@ module Docs · 3 unmodified lines
4	4
5	5	self.name = 'Ruby on Rails'
6	6	self.slug = 'rails'
7		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Rails'
8	7	self.initial_paths = %w(guides/index.html)
9	8	self.links = {
10	9	home: 'http://rubyonrails.org/',

lib/docs/scrapers/rdoc/ruby.rb+0−4

@@ -78,22 +78,18 @@ module Docs · 77 unmodified lines
78	78
79	79	version '2.5' do
80	80	self.release = '2.5.0'
81		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby25'
82	81	end
83	82
84	83	version '2.4' do
85	84	self.release = '2.4.3'
86		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby24'
87	85	end
88	86
89	87	version '2.3' do
90	88	self.release = '2.3.6'
91		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby23'
92	89	end
93	90
94	91	version '2.2' do
95	92	self.release = '2.2.9'
96		− self.dir = '/Users/Thibaut/DevDocs/Docs/RDoc/Ruby22'
97	93	end
98	94	end
99	95	end

lib/docs/scrapers/sqlite.rb+0−1

@@ -3,7 +3,6 @@ module Docs · 2 unmodified lines
3	3	self.name = 'SQLite'
4	4	self.type = 'sqlite'
5	5	self.release = '3.25.2'
6		− self.dir = '/Users/Thibaut/DevDocs/Docs/sqlite/'
7	6	self.base_url = 'https://sqlite.org/'
8	7	self.root_path = 'docs.html'
9	8	self.initial_paths = %w(keyword_index.html)

test/lib/docs/core/scrapers/file_scraper_test.rb+89−47

@@ -2,16 +2,23 @@ require 'test_helper' · 1 unmodified line
2	2	require 'docs'
3	3
4	4	class FileScraperTest < MiniTest::Spec
	5	+ ROOT_PATH = File.expand_path('../../../../../../', __FILE__)
	6	+
5	7	class Scraper < Docs::FileScraper
6		− self.dir = '/'
7	8	self.html_filters = Docs::FilterStack.new
8	9	self.text_filters = Docs::FilterStack.new
	10	+
	11	+ version 'version' do; end
9	12	end
10	13
11	14	let :scraper do
12	15	Scraper.new
13	16	end
14	17
	18	+ let :versioned_scraper do
	19	+ Scraper.versions.first.new
	20	+ end
	21	+
15	22	let :response do
16	23	OpenStruct.new body: 'body', url: Docs::URL.parse(Scraper.base_url)
17	24	end
@@ -22,9 +29,16 @@ class FileScraperTest < MiniTest::Spec · 4 unmodified lines
22	29	end
23	30	end
24	31
	32	+ describe "#source_directory" do
	33	+ it "returns the directory at docs/[slug]" do
	34	+ assert_equal File.join(ROOT_PATH, 'docs', 'scraper'), scraper.source_directory
	35	+ assert_equal File.join(ROOT_PATH, 'docs', 'scraper~version'), versioned_scraper.source_directory
	36	+ end
	37	+ end
	38	+
25	39	describe "#request_one" do
26	40	let :path do
27		− File.join(Scraper.dir, 'path')
	41	+ 'path'
28	42	end
29	43
30	44	let :result do
@@ -35,20 +49,34 @@ class FileScraperTest < MiniTest::Spec · 4 unmodified lines
35	49	stub(scraper).read_file
36	50	end
37	51
38		− it "reads a file" do
39		− mock(scraper).read_file(path)
40		− result
	52	+ context "when the source directory doesn't exist" do
	53	+ it "raises an error" do
	54	+ assert_raises Docs::SetupError do
	55	+ result
	56	+ end
	57	+ end
41	58	end
42	59
43		− describe "the returned response object" do
44		− it "has a #body" do
45		− stub(scraper).read_file { 'body' }
46		− assert_equal 'body', result.body
	60	+ context "when the source directory exists" do
	61	+ before do
	62	+ stub(scraper).assert_source_directory_exists
	63	+ end
	64	+
	65	+ it "reads a file" do
	66	+ mock(scraper).read_file(path)
	67	+ result
47	68	end
48	69
49		− it "has a #url" do
50		− assert_equal path, result.url.to_s
51		− assert_instance_of Docs::URL, result.url
	70	+ describe "the returned response object" do
	71	+ it "has a #body" do
	72	+ stub(scraper).read_file { 'body' }
	73	+ assert_equal 'body', result.body
	74	+ end
	75	+
	76	+ it "has a #url" do
	77	+ assert_equal path, result.url.to_s
	78	+ assert_instance_of Docs::URL, result.url
	79	+ end
52	80	end
53	81	end
54	82	end
@@ -58,49 +86,63 @@ class FileScraperTest < MiniTest::Spec · 3 unmodified lines
58	86	%w(one two)
59	87	end
60	88
61		− it "requests the given url" do
62		− mock(scraper).request_one('url')
63		− scraper.send(:request_all, 'url') {}
64		− end
65		−
66		− it "requests the given urls" do
67		− requests = []
68		− stub(scraper).request_one { \|url\| requests << url; nil }
69		− scraper.send(:request_all, urls) {}
70		− assert_equal urls, requests
71		− end
72		−
73		− it "yields the responses" do
74		− responses = []
75		− stub(scraper).request_one { \|url\| urls.index(url) }
76		− scraper.send(:request_all, urls) { \|response\| responses << response; nil }
77		− assert_equal (0...urls.length).to_a, responses
	89	+ context "when the source directory doesn't exist" do
	90	+ it "raises an error" do
	91	+ assert_raises Docs::SetupError do
	92	+ scraper.send(:request_all, urls) {}
	93	+ end
	94	+ end
78	95	end
79	96
80		− context "when the block returns an array" do
81		− let :next_urls do
82		− %w(three four)
	97	+ context "when the source directory exists" do
	98	+ before do
	99	+ stub(scraper).assert_source_directory_exists
83	100	end
84	101
85		− let :all_urls do
86		− urls + %w(three four)
	102	+ it "requests the given url" do
	103	+ mock(scraper).request_one('url')
	104	+ scraper.send(:request_all, 'url') {}
87	105	end
88	106
89		− it "requests the returned urls" do
	107	+ it "requests the given urls" do
90	108	requests = []
91		− stub(scraper).request_one { \|url\| requests << url; url }
92		− scraper.send(:request_all, urls) { [next_urls.shift].compact }
93		− assert_equal all_urls, requests
	109	+ stub(scraper).request_one { \|url\| requests << url; nil }
	110	+ scraper.send(:request_all, urls) {}
	111	+ assert_equal urls, requests
94	112	end
95	113
96		− it "yields their responses" do
	114	+ it "yields the responses" do
97	115	responses = []
98		− stub(scraper).request_one { \|url\| all_urls.index(url) }
99		− scraper.send :request_all, urls do \|response\|
100		− responses << response
101		− [next_urls.shift].compact
	116	+ stub(scraper).request_one { \|url\| urls.index(url) }
	117	+ scraper.send(:request_all, urls) { \|response\| responses << response; nil }
	118	+ assert_equal (0...urls.length).to_a, responses
	119	+ end
	120	+
	121	+ context "when the block returns an array" do
	122	+ let :next_urls do
	123	+ %w(three four)
	124	+ end
	125	+
	126	+ let :all_urls do
	127	+ urls + %w(three four)
	128	+ end
	129	+
	130	+ it "requests the returned urls" do
	131	+ requests = []
	132	+ stub(scraper).request_one { \|url\| requests << url; url }
	133	+ scraper.send(:request_all, urls) { [next_urls.shift].compact }
	134	+ assert_equal all_urls, requests
	135	+ end
	136	+
	137	+ it "yields their responses" do
	138	+ responses = []
	139	+ stub(scraper).request_one { \|url\| all_urls.index(url) }
	140	+ scraper.send :request_all, urls do \|response\|
	141	+ responses << response
	142	+ [next_urls.shift].compact
	143	+ end
	144	+ assert_equal (0...all_urls.length).to_a, responses
102	145	end
103		− assert_equal (0...all_urls.length).to_a, responses
104	146	end
105	147	end
106	148	end
@@ -126,13 +168,13 @@ class FileScraperTest < MiniTest::Spec · 19 unmodified lines
126	168	scraper.send :read_file, 'file'
127	169	end
128	170
129		− it "returns the file's content when the file exists" do
130		− stub(File).read('file') { 'content' }
	171	+ it "returns the file's content when the file exists in the source directory" do
	172	+ stub(File).read(File.join(ROOT_PATH, 'docs', 'scraper', 'file')) { 'content' }
131	173	assert_equal 'content', result
132	174	end
133	175
134	176	it "returns nil when the file doesn't exist" do
135		− stub(File).read('file') { raise }
	177	+ stub(File).read(File.join(ROOT_PATH, 'docs', 'scraper', 'file')) { raise }
136	178	assert_nil result
137	179	end
138	180	end