Unified · Split
| @@ -13,6 +13,8 @@ module Docs · 12 unmodified lines | ||
| 13 | 13 | end |
| 14 | 14 | end |
| 15 | 15 | |
| 16 | + @@rate_limiter = nil | |
| 17 | + | |
| 16 | 18 | self.params = {} |
| 17 | 19 | self.headers = { 'User-Agent' => 'DevDocs' } |
| 18 | 20 | self.force_gzip = false |
| @@ -24,6 +26,15 @@ module Docs · 5 unmodified lines | ||
| 24 | 26 | end |
| 25 | 27 | |
| 26 | 28 | def request_all(urls, &block) |
| 29 | + if options[:rate_limit] | |
| 30 | + if @@rate_limiter | |
| 31 | + @@rate_limiter.limit = options[:rate_limit] | |
| 32 | + else | |
| 33 | + @@rate_limiter = RateLimiter.new(options[:rate_limit]) | |
| 34 | + Typhoeus.before(&@@rate_limiter.to_proc) | |
| 35 | + end | |
| 36 | + end | |
| 37 | + | |
| 27 | 38 | Requester.run urls, request_options: request_options, &block |
| 28 | 39 | end |
| 29 | 40 | |
| @@ -165,5 +176,35 @@ module Docs · 135 unmodified lines | ||
| 165 | 176 | super.merge! redirections: self.class.redirections |
| 166 | 177 | end |
| 167 | 178 | end |
| 179 | + | |
| 180 | + class RateLimiter | |
| 181 | + attr_accessor :limit | |
| 182 | + | |
| 183 | + def initialize(limit) | |
| 184 | + @limit = limit | |
| 185 | + @minute = nil | |
| 186 | + @counter = 0 | |
| 187 | + end | |
| 188 | + | |
| 189 | + def call(*) | |
| 190 | + if @minute != Time.now.min | |
| 191 | + @minute = Time.now.min | |
| 192 | + @counter = 0 | |
| 193 | + end | |
| 194 | + | |
| 195 | + @counter += 1 | |
| 196 | + | |
| 197 | + if @counter >= @limit | |
| 198 | + wait = Time.now.end_of_minute.to_i - Time.now.to_i + 1 | |
| 199 | + sleep wait | |
| 200 | + end | |
| 201 | + | |
| 202 | + true | |
| 203 | + end | |
| 204 | + | |
| 205 | + def to_proc | |
| 206 | + method(:call).to_proc | |
| 207 | + end | |
| 208 | + end | |
| 168 | 209 | end |
| 169 | 210 | end |