Unified · Split
| @@ -13,6 +13,8 @@ module Docs · 12 unmodified lines | |||
| 13 | end | 13 | end |
| 14 | end | 14 | end |
| 15 | 15 | ||
| 16 | @@rate_limiter = nil | ||
| 17 | |||
| 16 | self.params = {} | 18 | self.params = {} |
| 17 | self.headers = { 'User-Agent' => 'DevDocs' } | 19 | self.headers = { 'User-Agent' => 'DevDocs' } |
| 18 | self.force_gzip = false | 20 | self.force_gzip = false |
| @@ -24,6 +26,15 @@ module Docs · 5 unmodified lines | |||
| 24 | end | 26 | end |
| 25 | 27 | ||
| 26 | def request_all(urls, &block) | 28 | def request_all(urls, &block) |
| 29 | if options[:rate_limit] | ||
| 30 | if @@rate_limiter | ||
| 31 | @@rate_limiter.limit = options[:rate_limit] | ||
| 32 | else | ||
| 33 | @@rate_limiter = RateLimiter.new(options[:rate_limit]) | ||
| 34 | Typhoeus.before(&@@rate_limiter.to_proc) | ||
| 35 | end | ||
| 36 | end | ||
| 37 | |||
| 27 | Requester.run urls, request_options: request_options, &block | 38 | Requester.run urls, request_options: request_options, &block |
| 28 | end | 39 | end |
| 29 | 40 | ||
| @@ -165,5 +176,35 @@ module Docs · 135 unmodified lines | |||
| 165 | super.merge! redirections: self.class.redirections | 176 | super.merge! redirections: self.class.redirections |
| 166 | end | 177 | end |
| 167 | end | 178 | end |
| 179 | |||
| 180 | class RateLimiter | ||
| 181 | attr_accessor :limit | ||
| 182 | |||
| 183 | def initialize(limit) | ||
| 184 | @limit = limit | ||
| 185 | @minute = nil | ||
| 186 | @counter = 0 | ||
| 187 | end | ||
| 188 | |||
| 189 | def call(*) | ||
| 190 | if @minute != Time.now.min | ||
| 191 | @minute = Time.now.min | ||
| 192 | @counter = 0 | ||
| 193 | end | ||
| 194 | |||
| 195 | @counter += 1 | ||
| 196 | |||
| 197 | if @counter >= @limit | ||
| 198 | wait = Time.now.end_of_minute.to_i - Time.now.to_i + 1 | ||
| 199 | sleep wait | ||
| 200 | end | ||
| 201 | |||
| 202 | true | ||
| 203 | end | ||
| 204 | |||
| 205 | def to_proc | ||
| 206 | method(:call).to_proc | ||
| 207 | end | ||
| 208 | end | ||
| 168 | end | 209 | end |
| 169 | end | 210 | end |