The solution that I went with was to use signals to trigger processing the next entry:
def perform
@hydra = Typhoeus::Hydra.new(:max_concurrency => 100)
set_trap
@target_concurrency.times { Process.kill("USR1", Process.pid) }
end
def set_trap
Signal.trap('USR1') do
if id = $redis.lpop(REDIS_UNSCRAPED_PRODUCTS_KEY)
product = Product.find(id).url
req = Typhoeus::Request.new(product.url, {:follow_location => true})
req.on_complete do |resp|
process_product_response(product, resp.body)
end
@hydra.queue(req)
end
end
end
def process_product_response(product, response)
# process product
Process.kill("USR1", Process.pid)
end