Implement initial_paths scraper option

Thibaut committed 12 years ago

Commit f8298181 parent 5e69d6d2 files · +79 −20Browse files

Unified · Split

2 files

lib/docs/core/scraper.rb +19 −8
test/lib/docs/core/scraper_test.rb +60 −12

@@ -4,7 +4,7 @@ require 'html/pipeline' · 3 unmodified lines
4	4	module Docs
5	5	class Scraper < Doc
6	6	class << self
7		− attr_accessor :base_url, :root_path, :html_filters, :text_filters, :options
	7	+ attr_accessor :base_url, :root_path, :initial_paths, :options, :html_filters, :text_filters
8	8
9	9	def inherited(subclass)
10	10	super
@@ -15,6 +15,7 @@ module Docs · 4 unmodified lines
15	15	end
16	16
17	17	subclass.root_path = root_path
	18	+ subclass.initial_paths = initial_paths.dup
18	19	subclass.options = options.deep_dup
19	20	subclass.html_filters = html_filters.inheritable_copy
20	21	subclass.text_filters = text_filters.inheritable_copy
@@ -27,11 +28,12 @@ module Docs · 6 unmodified lines
27	28
28	29	include Instrumentable
29	30
	31	+ self.initial_paths = []
	32	+ self.options = {}
	33	+
30	34	self.html_filters = FilterStack.new
31	35	self.text_filters = FilterStack.new
32	36
33		− self.options = {}
34		−
35	37	html_filters.push 'container', 'clean_html', 'normalize_urls', 'internal_urls', 'normalize_paths'
36	38	text_filters.push 'inner_html', 'clean_text', 'attribution'
37	39
@@ -43,10 +45,10 @@ module Docs · 5 unmodified lines
43	45	end
44	46
45	47	def build_pages
46		− requested_urls = Set.new [root_url.to_s.downcase]
47		− instrument 'running.scraper', urls: requested_urls.to_a
	48	+ requested_urls = Set.new initial_urls.map(&:downcase)
	49	+ instrument 'running.scraper', urls: initial_urls
48	50
49		− request_all root_url.to_s do \|response\|
	51	+ request_all initial_urls do \|response\|
50	52	next unless data = handle_response(response)
51	53	yield data
52	54	next unless data[:internal_urls].present?
@@ -72,6 +74,14 @@ module Docs · 19 unmodified lines
72	74	root_path.present? && root_path != '/'
73	75	end
74	76
	77	+ def initial_paths
	78	+ self.class.initial_paths
	79	+ end
	80	+
	81	+ def initial_urls
	82	+ @initial_urls \|\|= [root_url.to_s].concat(initial_paths.map(&method(:url_for))).freeze
	83	+ end
	84	+
75	85	def pipeline
76	86	@pipeline \|\|= ::HTML::Pipeline.new(self.class.filters).tap do \|pipeline\|
77	87	pipeline.instrumentation_service = Docs
@@ -80,14 +90,15 @@ module Docs · 2 unmodified lines
80	90
81	91	def options
82	92	@options \|\|= self.class.options.deep_dup.tap do \|options\|
83		− options.merge! base_url: base_url, root_url: root_url, root_path: root_path
	93	+ options.merge! base_url: base_url, root_url: root_url,
	94	+ root_path: root_path, initial_paths: initial_paths
84	95
85	96	if root_path?
86	97	(options[:skip] \|\|= []).concat ['', '/']
87	98	end
88	99
89	100	if options[:only] \|\| options[:only_patterns]
90		− (options[:only] \|\|= []).concat root_path? ? [root_path] : ['', '/']
	101	+ (options[:only] \|\|= []).concat initial_paths + (root_path? ? [root_path] : ['', '/'])
91	102	end
92	103
93	104	options.freeze

test/lib/docs/core/scraper_test.rb+60−12

@@ -6,6 +6,7 @@ class DocsScraperTest < MiniTest::Spec · 5 unmodified lines
6	6	self.type = 'scraper'
7	7	self.base_url = 'http://example.com/'
8	8	self.root_path = '/root'
	9	+ self.initial_paths = ['/initial']
9	10	self.html_filters = Docs::FilterStack.new
10	11	self.text_filters = Docs::FilterStack.new
11	12	end
@@ -37,6 +38,12 @@ class DocsScraperTest < MiniTest::Spec · 25 unmodified lines
37	38	assert_equal Scraper.root_path, subclass.root_path
38	39	end
39	40
	41	+ it "duplicates .initial_paths" do
	42	+ stub(Scraper).initial_paths { ['path'] }
	43	+ assert_equal Scraper.initial_paths, subclass.initial_paths
	44	+ refute_same Scraper.initial_paths, subclass.initial_paths
	45	+ end
	46	+
40	47	it "duplicates .options" do
41	48	stub(Scraper).options { { test: [] } }
42	49	assert_equal Scraper.options, subclass.options
@@ -112,6 +119,29 @@ class DocsScraperTest < MiniTest::Spec · 69 unmodified lines
112	119	end
113	120	end
114	121
	122	+ describe "#initial_urls" do
	123	+ let :initial_urls do
	124	+ scraper.initial_urls
	125	+ end
	126	+
	127	+ it "returns a frozen, memoized Array" do
	128	+ assert_instance_of Array, initial_urls
	129	+ assert initial_urls.frozen?
	130	+ assert_same initial_urls, scraper.initial_urls
	131	+ end
	132	+
	133	+ it "includes the #root_url" do
	134	+ assert_includes initial_urls, scraper.root_url.to_s
	135	+ end
	136	+
	137	+ it "includes the .initial_paths converted to urls" do
	138	+ stub(Scraper).base_url { 'http://example.com/' }
	139	+ stub(Scraper).initial_paths { ['one', '/two'] }
	140	+ assert_includes initial_urls, 'http://example.com/one'
	141	+ assert_includes initial_urls, 'http://example.com/two'
	142	+ end
	143	+ end
	144	+
115	145	describe "#build_page" do
116	146	before do
117	147	stub(scraper).handle_response
@@ -171,8 +201,8 @@ class DocsScraperTest < MiniTest::Spec · 53 unmodified lines
171	201	Proc.new {}
172	202	end
173	203
174		− it "requests the root url" do
175		− mock(scraper).request_all(scraper.root_url.to_s)
	204	+ it "requests the #initial_urls" do
	205	+ mock(scraper).request_all(scraper.initial_urls)
176	206	scraper.build_pages(&block)
177	207	end
178	208
@@ -181,12 +211,14 @@ class DocsScraperTest < MiniTest::Spec · 2 unmodified lines
181	211	scraper.build_pages(&block)
182	212	assert scraper.last_instrumentation
183	213	assert_equal 'running.scraper', scraper.last_instrumentation[:event]
184		− assert_includes scraper.last_instrumentation[:payload][:urls], scraper.root_url.to_s
	214	+ assert_equal scraper.initial_urls, scraper.last_instrumentation[:payload][:urls]
185	215	end
186	216
187	217	context "when the response is processable" do
188	218	before do
189		− stub(scraper).request_all.with_any_args { \|*args\| @returned = args.last.call(response) }
	219	+ stub(scraper).request_all do \|urls, block\|
	220	+ urls.each { \|url\| @next_urls \|\|= block.call(response) }
	221	+ end
190	222	stub(scraper).handle_response(response) { processed_response }
191	223	end
192	224
@@ -203,7 +235,7 @@ class DocsScraperTest < MiniTest::Spec · 10 unmodified lines
203	235
204	236	it "requests nothing more" do
205	237	scraper.build_pages(&block)
206		− assert_nil @returned
	238	+ assert_nil @next_urls
207	239	end
208	240
209	241	it "doesn't instrument 'queued'" do
@@ -223,14 +255,14 @@ class DocsScraperTest < MiniTest::Spec · 13 unmodified lines
223	255
224	256	it "requests the urls" do
225	257	scraper.build_pages(&block)
226		− assert_equal urls, @returned
	258	+ assert_equal urls, @next_urls
227	259	end
228	260
229	261	it "doesn't request the same url twice irrespective of case" do
230	262	stub(Scraper).root_path { 'PATH' }
231		− processed_response[:internal_urls] = [scraper.root_url.to_s.swapcase]
	263	+ processed_response[:internal_urls] = scraper.initial_urls.map(&:swapcase)
232	264	scraper.build_pages(&block)
233		− assert_empty @returned
	265	+ assert_empty @next_urls
234	266	end
235	267
236	268	it "instruments 'queued'" do
@@ -280,6 +312,10 @@ class DocsScraperTest < MiniTest::Spec · 43 unmodified lines
280	312	assert_equal '/root', options[:root_path]
281	313	end
282	314
	315	+ it "includes .initial_paths" do
	316	+ assert_equal ['/initial'], options[:initial_paths]
	317	+ end
	318	+
283	319	context "when #root_path? is false" do
284	320	before do
285	321	stub(scraper).root_path? { false }
@@ -300,15 +336,27 @@ class DocsScraperTest < MiniTest::Spec · 14 unmodified lines
300	336	assert_includes options[:only], '/'
301	337	end
302	338
	339	+ it "adds .initial_paths" do
	340	+ assert_includes options[:only], '/initial'
	341	+ end
	342	+
303	343	it "doesn't modify the array in place" do
304	344	assert_equal ['/path'], Scraper.options[:only]
305	345	end
306	346	end
307	347
308	348	context "and :only_patterns is an array" do
309		− it "assigns ['', '/'] to :only" do
	349	+ before do
310	350	stub(Scraper).options { { only_patterns: [] } }
311		− assert_equal ['', '/'], options[:only]
	351	+ end
	352	+
	353	+ it "adds ['', '/'] to :only" do
	354	+ assert_includes options[:only], ''
	355	+ assert_includes options[:only], '/'
	356	+ end
	357	+
	358	+ it "adds .initial_paths to :only" do
	359	+ assert_includes options[:only], '/initial'
312	360	end
313	361	end
314	362	end
@@ -348,9 +396,9 @@ class DocsScraperTest < MiniTest::Spec · 33 unmodified lines
348	396	end
349	397
350	398	context "and :only_patterns is an array" do
351		− it "assigns [.root_path] to :only" do
	399	+ it "adds .root_path to :only" do
352	400	stub(Scraper).options { { only_patterns: [] } }
353		− assert_equal ['/root'], options[:only]
	401	+ assert_includes options[:only], '/root'
354	402	end
355	403	end
356	404	end