| @@ -21,10 +21,6 @@ class DocsScraperTest < MiniTest::Spec · 20 unmodified lines |
| 21 | Struct.new(:body, :url).new | 21 | Struct.new(:body, :url).new |
| 22 | end | 22 | end |
| 23 | | 23 | |
| 24 | let :processed_response do | | |
| 25 | Hash.new | | |
| 26 | end | | |
| 27 | | | |
| 28 | describe ".inherited" do | 24 | describe ".inherited" do |
| 29 | let :subclass do | 25 | let :subclass do |
| 30 | Class.new Scraper | 26 | Class.new Scraper |
| @@ -88,33 +84,40 @@ class DocsScraperTest < MiniTest::Spec · 57 unmodified lines |
| 88 | end | 84 | end |
| 89 | | 85 | |
| 90 | describe "#root_url" do | 86 | describe "#root_url" do |
| | 87 | let :root_url do |
| | 88 | scraper.root_url |
| | 89 | end |
| | 90 | |
| 91 | context "when #root_path? is false" do | 91 | context "when #root_path? is false" do |
| 92 | before do | 92 | before do |
| 93 | stub(scraper).root_path? { false } | 93 | stub(scraper).root_path? { false } |
| 94 | end | 94 | end |
| 95 | | 95 | |
| 96 | it "returns a Docs::URL" do | 96 | it "returns a memoized Docs::URL" do |
| 97 | assert_instance_of Docs::URL, scraper.root_url | 97 | assert_instance_of Docs::URL, root_url |
| | 98 | assert_same root_url, scraper.root_url |
| 98 | end | 99 | end |
| 99 | | 100 | |
| 100 | it "returns the normalized base url" do | 101 | it "returns the normalized .base_url" do |
| 101 | stub(Scraper).base_url { 'http://example.com' } | 102 | stub(Scraper).base_url { 'http://example.com' } |
| 102 | assert_equal 'http://example.com/', scraper.root_url.to_s | 103 | assert_equal 'http://example.com/', root_url.to_s |
| 103 | end | 104 | end |
| 104 | end | 105 | end |
| 105 | | 106 | |
| 106 | context "when .root_path isn't blank" do | 107 | context "when #root_path? is true" do |
| 107 | before do | 108 | before do |
| 108 | stub(scraper).root_path? { true } | 109 | stub(scraper).root_path? { true } |
| 109 | end | 110 | end |
| 110 | | 111 | |
| 111 | it "returns a Docs::URL" do | 112 | it "returns a memoized Docs::URL" do |
| 112 | assert_instance_of Docs::URL, scraper.root_url | 113 | assert_instance_of Docs::URL, root_url |
| | 114 | assert_same root_url, scraper.root_url |
| 113 | end | 115 | end |
| 114 | | 116 | |
| 115 | it "returns base url + root path" do | 117 | it "returns .base_url + .root_path" do |
| 116 | stub(Scraper).base_url { 'http://example.com/path/' } | 118 | stub(Scraper).base_url { 'http://example.com/path/' } |
| 117 | assert_equal 'http://example.com/path/root', scraper.root_url.to_s | 119 | stub(Scraper).root_path { '/root' } |
| | 120 | assert_equal 'http://example.com/path/root', root_url.to_s |
| 118 | end | 121 | end |
| 119 | end | 122 | end |
| 120 | end | 123 | end |
| @@ -201,6 +204,10 @@ class DocsScraperTest < MiniTest::Spec · 80 unmodified lines |
| 201 | Proc.new {} | 204 | Proc.new {} |
| 202 | end | 205 | end |
| 203 | | 206 | |
| | 207 | let :processed_response do |
| | 208 | Hash.new |
| | 209 | end |
| | 210 | |
| 204 | it "requests the #initial_urls" do | 211 | it "requests the #initial_urls" do |
| 205 | mock(scraper).request_all(scraper.initial_urls) | 212 | mock(scraper).request_all(scraper.initial_urls) |
| 206 | scraper.build_pages(&block) | 213 | scraper.build_pages(&block) |
| @@ -224,11 +231,10 @@ class DocsScraperTest < MiniTest::Spec · 17 unmodified lines |
| 224 | | 231 | |
| 225 | it "yields the processed response" do | 232 | it "yields the processed response" do |
| 226 | scraper.build_pages { |arg| @arg = arg } | 233 | scraper.build_pages { |arg| @arg = arg } |
| 227 | assert @arg | 234 | assert_same processed_response, @arg |
| 228 | assert_equal processed_response, @arg | | |
| 229 | end | 235 | end |
| 230 | | 236 | |
| 231 | context "when response[:internal_urls] is empty" do | 237 | context "when :internal_urls is empty" do |
| 232 | before do | 238 | before do |
| 233 | processed_response[:internal_urls] = [] | 239 | processed_response[:internal_urls] = [] |
| 234 | end | 240 | end |
| @@ -244,22 +250,21 @@ class DocsScraperTest < MiniTest::Spec · 9 unmodified lines |
| 244 | end | 250 | end |
| 245 | end | 251 | end |
| 246 | | 252 | |
| 247 | context "when response[:internal_urls] isn't empty" do | 253 | context "when :internal_urls isn't empty" do |
| 248 | let :urls do | 254 | let :internal_urls do |
| 249 | ['Url'] | 255 | ['Url'] |
| 250 | end | 256 | end |
| 251 | | 257 | |
| 252 | before do | 258 | before do |
| 253 | processed_response[:internal_urls] = urls | 259 | processed_response[:internal_urls] = internal_urls |
| 254 | end | 260 | end |
| 255 | | 261 | |
| 256 | it "requests the urls" do | 262 | it "requests the urls" do |
| 257 | scraper.build_pages(&block) | 263 | scraper.build_pages(&block) |
| 258 | assert_equal urls, @next_urls | 264 | assert_equal internal_urls, @next_urls |
| 259 | end | 265 | end |
| 260 | | 266 | |
| 261 | it "doesn't request the same url twice irrespective of case" do | 267 | it "doesn't request the same url twice irrespective of case" do |
| 262 | stub(Scraper).root_path { 'PATH' } | | |
| 263 | processed_response[:internal_urls] = scraper.initial_urls.map(&:swapcase) | 268 | processed_response[:internal_urls] = scraper.initial_urls.map(&:swapcase) |
| 264 | scraper.build_pages(&block) | 269 | scraper.build_pages(&block) |
| 265 | assert_empty @next_urls | 270 | assert_empty @next_urls |
| @@ -269,7 +274,7 @@ class DocsScraperTest < MiniTest::Spec · 3 unmodified lines |
| 269 | scraper.build_pages(&block) | 274 | scraper.build_pages(&block) |
| 270 | assert scraper.last_instrumentation | 275 | assert scraper.last_instrumentation |
| 271 | assert_equal 'queued.scraper', scraper.last_instrumentation[:event] | 276 | assert_equal 'queued.scraper', scraper.last_instrumentation[:event] |
| 272 | assert_equal urls, scraper.last_instrumentation[:payload][:urls] | 277 | assert_equal internal_urls, scraper.last_instrumentation[:payload][:urls] |
| 273 | end | 278 | end |
| 274 | end | 279 | end |
| 275 | end | 280 | end |
| @@ -286,34 +291,59 @@ class DocsScraperTest < MiniTest::Spec · 10 unmodified lines |
| 286 | | 291 | |
| 287 | describe "#options" do | 292 | describe "#options" do |
| 288 | let :options do | 293 | let :options do |
| | 294 | Hash.new |
| | 295 | end |
| | 296 | |
| | 297 | let :result do |
| 289 | scraper.options | 298 | scraper.options |
| 290 | end | 299 | end |
| 291 | | 300 | |
| | 301 | before do |
| | 302 | stub(Scraper).options { options } |
| | 303 | end |
| | 304 | |
| 292 | it "returns a frozen, memoized Hash" do | 305 | it "returns a frozen, memoized Hash" do |
| 293 | assert_instance_of Hash, options | 306 | assert_instance_of Hash, result |
| 294 | assert options.frozen? | 307 | assert result.frozen? |
| 295 | assert_same options, scraper.options | 308 | assert_same result, scraper.options |
| 296 | end | 309 | end |
| 297 | | 310 | |
| 298 | it "includes .options" do | 311 | it "includes .options" do |
| 299 | stub(Scraper).options { { test: true } } | 312 | options[:test] = true |
| 300 | assert options[:test] | 313 | assert result[:test] |
| 301 | end | 314 | end |
| 302 | | 315 | |
| 303 | it "includes #base_url" do | 316 | it "includes #base_url" do |
| 304 | assert_equal scraper.base_url, options[:base_url] | 317 | assert_equal scraper.base_url, result[:base_url] |
| 305 | end | 318 | end |
| 306 | | 319 | |
| 307 | it "includes #root_url" do | 320 | it "includes #root_url" do |
| 308 | assert_equal scraper.root_url, options[:root_url] | 321 | assert_equal scraper.root_url, result[:root_url] |
| | 322 | end |
| | 323 | |
| | 324 | it "includes #root_path" do |
| | 325 | assert_equal '/root', result[:root_path] |
| | 326 | end |
| | 327 | |
| | 328 | it "includes #initial_paths" do |
| | 329 | assert_equal ['/initial'], result[:initial_paths] |
| | 330 | end |
| | 331 | |
| | 332 | it "adds #initial_paths to :only when it is an array" do |
| | 333 | options[:only] = ['/path'] |
| | 334 | assert_includes result[:only], options[:only].first |
| | 335 | assert_includes result[:only], '/initial' |
| 309 | end | 336 | end |
| 310 | | 337 | |
| 311 | it "includes .root_path" do | 338 | it "adds #initial_paths to :only when :only_patterns is an array" do |
| 312 | assert_equal '/root', options[:root_path] | 339 | options[:only_patterns] = [] |
| | 340 | assert_includes result[:only], '/initial' |
| 313 | end | 341 | end |
| 314 | | 342 | |
| 315 | it "includes .initial_paths" do | 343 | it "doesn't modify :only in place" do |
| 316 | assert_equal ['/initial'], options[:initial_paths] | 344 | options[:only] = [] |
| | 345 | result |
| | 346 | assert_empty options[:only] |
| 317 | end | 347 | end |
| 318 | | 348 | |
| 319 | context "when #root_path? is false" do | 349 | context "when #root_path? is false" do |
| @@ -322,42 +352,27 @@ class DocsScraperTest < MiniTest::Spec · 2 unmodified lines |
| 322 | end | 352 | end |
| 323 | | 353 | |
| 324 | it "doesn't modify :skip" do | 354 | it "doesn't modify :skip" do |
| 325 | assert_nil options[:skip] | 355 | options[:skip] = [] |
| | 356 | assert_equal options[:skip], result[:skip] |
| 326 | end | 357 | end |
| 327 | | 358 | |
| 328 | context "and :only is an array" do | 359 | it "adds '' and '/' to :only when it is an array" do |
| 329 | before do | 360 | options[:only] = ['/path'] |
| 330 | stub(Scraper).options { { only: ['/path'] } } | 361 | assert_includes result[:only], options[:only].first |
| 331 | end | 362 | assert_includes result[:only], '' |
| 332 | | 363 | assert_includes result[:only], '/' |
| 333 | it "adds ['', '/']" do | | |
| 334 | assert_includes options[:only], '/path' | | |
| 335 | assert_includes options[:only], '' | | |
| 336 | assert_includes options[:only], '/' | | |
| 337 | end | | |
| 338 | | | |
| 339 | it "adds .initial_paths" do | | |
| 340 | assert_includes options[:only], '/initial' | | |
| 341 | end | | |
| 342 | | | |
| 343 | it "doesn't modify the array in place" do | | |
| 344 | assert_equal ['/path'], Scraper.options[:only] | | |
| 345 | end | | |
| 346 | end | 364 | end |
| 347 | | 365 | |
| 348 | context "and :only_patterns is an array" do | 366 | it "adds '' and '/' to :only when :only_patterns is an array" do |
| 349 | before do | 367 | options[:only_patterns] = [] |
| 350 | stub(Scraper).options { { only_patterns: [] } } | 368 | assert_includes result[:only], '' |
| 351 | end | 369 | assert_includes result[:only], '/' |
| 352 | | 370 | end |
| 353 | it "adds ['', '/'] to :only" do | | |
| 354 | assert_includes options[:only], '' | | |
| 355 | assert_includes options[:only], '/' | | |
| 356 | end | | |
| 357 | | 371 | |
| 358 | it "adds .initial_paths to :only" do | 372 | it "doesn't modify :only in place" do |
| 359 | assert_includes options[:only], '/initial' | 373 | options[:only] = [] |
| 360 | end | 374 | result |
| | 375 | assert_empty options[:only] |
| 361 | end | 376 | end |
| 362 | end | 377 | end |
| 363 | | 378 | |
| @@ -366,40 +381,33 @@ class DocsScraperTest < MiniTest::Spec · 2 unmodified lines |
| 366 | stub(scraper).root_path? { true } | 381 | stub(scraper).root_path? { true } |
| 367 | end | 382 | end |
| 368 | | 383 | |
| 369 | context "and :skip is nil" do | 384 | it "adds '' and '/' to :skip when it is nil" do |
| 370 | it "assigns it ['', '/']" do | 385 | assert_includes result[:skip], '' |
| 371 | assert_equal ['', '/'], options[:skip] | 386 | assert_includes result[:skip], '/' |
| 372 | end | | |
| 373 | end | 387 | end |
| 374 | | 388 | |
| 375 | context "and :skip is an array" do | 389 | it "adds '' and '/' to :skip when it is an array" do |
| 376 | before do | 390 | options[:skip] = ['/path'] |
| 377 | stub(Scraper).options { { skip: ['/path'] } } | 391 | assert_includes result[:skip], options[:skip].first |
| 378 | end | 392 | assert_includes result[:skip], '' |
| 379 | | 393 | assert_includes result[:skip], '/' |
| 380 | it "adds ['', '/']" do | 394 | end |
| 381 | assert_includes options[:skip], '/path' | | |
| 382 | assert_includes options[:skip], '' | | |
| 383 | assert_includes options[:skip], '/' | | |
| 384 | end | | |
| 385 | | 395 | |
| 386 | it "doesn't modify the array in place" do | 396 | it "doesn't modify :skip in place" do |
| 387 | assert_equal ['/path'], Scraper.options[:skip] | 397 | options[:skip] = [] |
| 388 | end | 398 | result |
| | 399 | assert_empty options[:skip] |
| 389 | end | 400 | end |
| 390 | | 401 | |
| 391 | context "and :only is an array" do | 402 | it "adds #root_path to :only when it is an array" do |
| 392 | it "adds .root_path" do | 403 | options[:only] = ['/path'] |
| 393 | stub(Scraper).options { { only: [] } } | 404 | assert_includes result[:only], options[:only].first |
| 394 | assert_includes options[:only], '/root' | 405 | assert_includes result[:only], '/root' |
| 395 | end | | |
| 396 | end | 406 | end |
| 397 | | 407 | |
| 398 | context "and :only_patterns is an array" do | 408 | it "adds #root_path to :only when :only_patterns is an array" do |
| 399 | it "adds .root_path to :only" do | 409 | options[:only_patterns] = [] |
| 400 | stub(Scraper).options { { only_patterns: [] } } | 410 | assert_includes result[:only], '/root' |
| 401 | assert_includes options[:only], '/root' | | |
| 402 | end | | |
| 403 | end | 411 | end |
| 404 | end | 412 | end |
| 405 | end | 413 | end |
| @@ -484,18 +492,22 @@ class DocsScraperTest < MiniTest::Spec · 78 unmodified lines |
| 484 | end | 492 | end |
| 485 | | 493 | |
| 486 | describe "#pipeline" do | 494 | describe "#pipeline" do |
| 487 | it "returns an HTML::Pipeline with .filters" do | 495 | let :pipeline do |
| 488 | stub(Scraper).filters { [1] } | 496 | scraper.pipeline |
| 489 | assert_instance_of ::HTML::Pipeline, scraper.pipeline | | |
| 490 | assert_equal Scraper.filters, scraper.pipeline.filters | | |
| 491 | end | 497 | end |
| 492 | | 498 | |
| 493 | it "is memoized" do | 499 | it "returns a memoized HTML::Pipeline" do |
| 494 | assert_same scraper.pipeline, scraper.pipeline | 500 | assert_instance_of ::HTML::Pipeline, pipeline |
| | 501 | assert_same pipeline, scraper.pipeline |
| | 502 | end |
| | 503 | |
| | 504 | it "returns a pipeline with the filters stored in .filters" do |
| | 505 | stub(Scraper).filters { [1] } |
| | 506 | assert_equal Scraper.filters, pipeline.filters |
| 495 | end | 507 | end |
| 496 | | 508 | |
| 497 | it "assigns Docs as the pipeline's instrumentation service" do | 509 | it "returns a pipeline with Docs as instrumentation service" do |
| 498 | assert_equal Docs, scraper.pipeline.instrumentation_service | 510 | assert_equal Docs, pipeline.instrumentation_service |
| 499 | end | 511 | end |
| 500 | end | 512 | end |
| 501 | end | 513 | end |