Object
# File lib/typhoeus/hydra.rb, line 18 18: def self.hydra 19: @hydra ||= new 20: end
# File lib/typhoeus/hydra.rb, line 22 22: def self.hydra=(val) 23: @hydra = val 24: end
# File lib/typhoeus/hydra.rb, line 3 3: def initialize(options = {}) 4: @memoize_requests = true 5: @multi = Multi.new 6: @easy_pool = [] 7: initial_pool_size = options[:initial_pool_size] || 10 8: @max_concurrency = options[:max_concurrency] || 200 9: initial_pool_size.times { @easy_pool << Easy.new } 10: @stubs = [] 11: @memoized_requests = {} 12: @retrieved_from_cache = {} 13: @queued_requests = [] 14: @running_requests = 0 15: @stubbed_request_count = 0 16: end
# File lib/typhoeus/hydra.rb, line 84 84: def cache_getter(&block) 85: @cache_getter = block 86: end
# File lib/typhoeus/hydra.rb, line 88 88: def cache_setter(&block) 89: @cache_setter = block 90: end
# File lib/typhoeus/hydra.rb, line 26 26: def clear_cache_callbacks 27: @cache_setter = nil 28: @cache_getter = nil 29: end
# File lib/typhoeus/hydra.rb, line 31 31: def clear_stubs 32: @stubs = [] 33: end
# File lib/typhoeus/hydra.rb, line 80 80: def disable_memoization 81: @memoize_requests = false 82: end
# File lib/typhoeus/hydra.rb, line 35 35: def fire_and_forget 36: @queued_requests.each {|r| queue(r, false)} 37: @multi.fire_and_forget 38: end
# File lib/typhoeus/hydra.rb, line 92 92: def on_complete(&block) 93: @on_complete = block 94: end
# File lib/typhoeus/hydra.rb, line 96 96: def on_complete=(proc) 97: @on_complete = proc 98: end
# File lib/typhoeus/hydra.rb, line 40 40: def queue(request, obey_concurrency_limit = true) 41: return if assign_to_stub(request) 42: 43: if @running_requests >= @max_concurrency && obey_concurrency_limit 44: @queued_requests << request 45: else 46: if request.method == :get 47: if @memoize_requests && @memoized_requests.has_key?(request.url) 48: if response = @retrieved_from_cache[request.url] 49: request.response = response 50: request.call_handlers 51: else 52: @memoized_requests[request.url] << request 53: end 54: else 55: @memoized_requests[request.url] = [] if @memoize_requests 56: get_from_cache_or_queue(request) 57: end 58: else 59: get_from_cache_or_queue(request) 60: end 61: end 62: end
# File lib/typhoeus/hydra.rb, line 64 64: def run 65: while @stubbed_request_count > 0 66: @stubs.each do |m| 67: while request = m.requests.shift 68: @stubbed_request_count -= 1 69: m.response.request = request 70: handle_request(request, m.response) 71: end 72: end 73: end 74: 75: @multi.perform 76: @memoized_requests = {} 77: @retrieved_from_cache = {} 78: end
# File lib/typhoeus/hydra.rb, line 105 105: def assign_to_stub(request) 106: m = @stubs.detect {|stub| stub.matches? request} 107: if m 108: m.add_request(request) 109: @stubbed_request_count += 1 110: else 111: nil 112: end 113: end
# File lib/typhoeus/hydra.rb, line 131 131: def get_easy_object(request) 132: @running_requests += 1 133: 134: easy = @easy_pool.pop || Easy.new 135: easy.verbose = request.verbose 136: if request.username || request.password 137: auth = { :username => request.username, :password => request.password } 138: auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method 139: easy.auth = auth 140: end 141: easy.url = request.url 142: easy.method = request.method 143: easy.params = request.params if request.method == :post && !request.params.nil? 144: easy.headers = request.headers if request.headers 145: easy.request_body = request.body if request.body 146: easy.timeout = request.timeout if request.timeout 147: easy.connect_timeout = request.connect_timeout if request.connect_timeout 148: easy.follow_location = request.follow_location if request.follow_location 149: easy.max_redirects = request.max_redirects if request.max_redirects 150: easy.proxy = request.proxy if request.proxy 151: easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification 152: easy.ssl_cert = request.ssl_cert 153: easy.ssl_cert_type = request.ssl_cert_type 154: easy.ssl_key = request.ssl_key 155: easy.ssl_key_type = request.ssl_key_type 156: easy.ssl_key_password = request.ssl_key_password 157: easy.ssl_cacert = request.ssl_cacert 158: easy.ssl_capath = request.ssl_capath 159: easy.verbose = request.verbose 160: 161: easy.on_success do |easy| 162: queue_next 163: handle_request(request, response_from_easy(easy, request)) 164: release_easy_object(easy) 165: end 166: easy.on_failure do |easy| 167: queue_next 168: handle_request(request, response_from_easy(easy, request)) 169: release_easy_object(easy) 170: end 171: easy.set_headers 172: easy 173: end
# File lib/typhoeus/hydra.rb, line 116 116: def get_from_cache_or_queue(request) 117: if @cache_getter 118: val = @cache_getter.call(request) 119: if val 120: @retrieved_from_cache[request.url] = val 121: handle_request(request, val, false) 122: else 123: @multi.add(get_easy_object(request)) 124: end 125: else 126: @multi.add(get_easy_object(request)) 127: end 128: end
# File lib/typhoeus/hydra.rb, line 188 188: def handle_request(request, response, live_request = true) 189: request.response = response 190: 191: if live_request && request.cache_timeout && @cache_setter 192: @cache_setter.call(request) 193: end 194: @on_complete.call(response) if @on_complete 195: 196: request.call_handlers 197: if requests = @memoized_requests[request.url] 198: requests.each do |r| 199: r.response = response 200: r.call_handlers 201: end 202: end 203: end
# File lib/typhoeus/hydra.rb, line 176 176: def queue_next 177: @running_requests -= 1 178: queue(@queued_requests.pop) unless @queued_requests.empty? 179: end
# File lib/typhoeus/hydra.rb, line 182 182: def release_easy_object(easy) 183: easy.reset 184: @easy_pool.push easy 185: end
# File lib/typhoeus/hydra.rb, line 206 206: def response_from_easy(easy, request) 207: Response.new(:code => easy.response_code, 208: :headers => easy.response_header, 209: :body => easy.response_body, 210: :time => easy.total_time_taken, 211: :effective_url => easy.effective_url, 212: :request => request) 213: end
Disabled; run with --debug to generate this.
Generated with the Darkfish Rdoc Generator 1.1.6.