Threading HTTP requests in Crystal

Question

I have code that needs to run in "parallel" (not real, I know Crystal does not support parallelism).

require "http/client"

thread_count = 4
resps = [] of HTTP::Client::Response
mutex = Thread::Mutex.new

urls = [] of String
(1..10).each { |i| urls << "http://httpbin.org/delay#{i}"}

threads = Array.new(thread_count) {
  Thread.new do
    while url = mutex.synchronize { urls.pop? }
      resp = HTTP::Client.get(url)
      mutex.synchronize { resps << resp }
    end
  end
}

threads.map(&.join)

The source file for the Thread class says not to use it. Anyway, this code does not work with HTTP::Client.

Answer 1

Use spawn:

require "http/client"
require "json"

WORKER_COUNT = 4

to_fetch = Channel(String?).new(WORKER_COUNT)
responses = Channel(HTTP::Client::Response?).new(WORKER_COUNT)

WORKER_COUNT.times do
  spawn do
    loop do
      url = to_fetch.receive
      responses.send url ? HTTP::Client.get(url) : nil
      break unless url
    end
  end
end

spawn do
  10.times do |i|
    to_fetch.send "http://httpbin.org/delay/#{i}"
  end
  WORKER_COUNT.times do
    to_fetch.send nil
  end
end

start = Time.local
worker_done_count = 0
loop do
  response = responses.receive
  if response
    puts "#{Time.local - start}: fetched #{JSON.parse(response.body)["url"]}"
  else
    worker_done_count += 1
    break if worker_done_count == WORKER_COUNT
  end
end