History

Specs all passing

Paul Sernatinger committed 4 years ago

Commit 2da05d71 parent b3e55ed37 files · +198 −178Browse files

Showing the changes to lib/docs (3 of 37 files). Show all

Unified · Split

3 files
  1. lib/docs/core/request.rb +1 −1
  2. lib/docs/core/requester.rb +8 −8
  3. lib/docs/core/scraper.rb +1 −1
lib/docs/core/request.rb+1−1
@@ -13,7 +13,7 @@ module Docs · 12 unmodified lines
13 request.run13 request.run
14 end14 end
15 15
16 def initialize(url, options = {})16 def initialize(url, **options)
17 super url.to_s, DEFAULT_OPTIONS.merge(options)17 super url.to_s, DEFAULT_OPTIONS.merge(options)
18 end18 end
19 19
lib/docs/core/requester.rb+8−8
@@ -4,10 +4,10 @@ module Docs · 3 unmodified lines
4 4
5 attr_reader :request_options5 attr_reader :request_options
6 6
7 def self.run(urls, options = {}, &block)7 def self.run(urls, **options, &block)
8 urls = urls.dup8 urls = urls.dup
9 requester = new(options)9 requester = new(**options)
10 requester.on_response(&block) if block10 requester.on_response(&block) if block_given?
11 requester.on_response do # cheap hack to ensure root page is processed first11 requester.on_response do # cheap hack to ensure root page is processed first
12 if urls12 if urls
13 requester.request(urls)13 requester.request(urls)
@@ -19,16 +19,16 @@ module Docs · 5 unmodified lines
19 requester19 requester
20 end20 end
21 21
22 def initialize(options = {})22 def initialize(**options)
23 @request_options = options.extract!(:request_options)[:request_options].try(:dup) || {}23 @request_options = options.extract!(:request_options)[:request_options].try(:dup) || {}
24 options[:max_concurrency] ||= 2024 options[:max_concurrency] ||= 20
25 options[:pipelining] = 025 options[:pipelining] = 0
26 super26 super
27 end27 end
28 28
29 def request(urls, options = {}, &block)29 def request(urls, **options, &block)
30 requests = [urls].flatten.map do |url|30 requests = [urls].flatten.map do |url|
31 build_and_queue_request(url, options, &block)31 build_and_queue_request(url, **options, &block)
32 end32 end
33 requests.length == 1 ? requests.first : requests33 requests.length == 1 ? requests.first : requests
34 end34 end
@@ -46,8 +46,8 @@ module Docs · 11 unmodified lines
46 46
47 private47 private
48 48
49 def build_and_queue_request(url, options, &block)49 def build_and_queue_request(url, **options, &block)
50 request = Request.new(url, request_options.merge(options))50 request = Request.new(url, **request_options.merge(options))
51 request.on_complete(&block) if block51 request.on_complete(&block) if block
52 queue(request)52 queue(request)
53 request53 request
lib/docs/core/scraper.rb+1−1
@@ -51,7 +51,7 @@ module Docs · 50 unmodified lines
51 end51 end
52 52
53 def initialize_stubs53 def initialize_stubs
54 self.class.stubs.each do |path, block|54 self.class.stubs.each do |path, &block|
55 Typhoeus.stub(url_for(path)).and_return do55 Typhoeus.stub(url_for(path)).and_return do
56 Typhoeus::Response.new \56 Typhoeus::Response.new \
57 effective_url: url_for(path),57 effective_url: url_for(path),