curl-downloader
Powerfull http-client for Crystal based on libcurl binding. LibCurl is much more powerfull than std crystal http-client, with features like: redirects, interface binding, socks, proxing, sessions, and many others. LibCurl is usually hard to use directly in event based languages like Crystal, because it block main thread. To avoid thread blocking you can use run_with_fork. LibCurl also have multi interface for concurrency, but in my expirience it quite bad and not managable (run_with_fork gives me much more concurrency).
Installation
$ brew install curl
$ apt-get install libcurl-dev
Installation
Add this to your application's shard.yml
:
dependencies:
curl-downloader:
github: kostya/curl-downloader
Usage
require "curl-downloader"
d = Curl::Downloader.new
d.url = "https://google.com/"
d.follow_redirects!
d.headers = {"User-Agent" => "Opera 9.51"}
d.timeout = 60
d.cookie_jar = "/tmp/downloader_test.txt"
d.cookie_file = "/tmp/downloader_test.txt"
d.execute
p d.code
p d.http_status
p d.content[0..100] + "..."
p d.headers.split("\r\n")
p d.url_effective
p d.content_type
d.free
Example run_with_fork
require "curl-downloader"
require "run_with_fork"
require "msgpack"
struct Response
MessagePack.mapping({
code: Int32,
http_status: Int32,
content: String,
})
end
pid, r = Process.run_with_fork do |w|
d = Curl::Downloader.new
d.url = "https://google.com/"
d.follow_redirects!
d.headers = {"User-Agent" => "Opera 9.51"}
d.timeout = 60
d.cookie_jar = "/tmp/downloader_test.txt"
d.cookie_file = "/tmp/downloader_test.txt"
d.execute
{code: d.code,
http_status: d.http_status,
content: d.content}.to_msgpack(w)
end
resp = Response.from_msgpack(r)
p resp
Example concurrent execution
require "curl-downloader"
require "run_with_fork"
def request(url)
d = Curl::Downloader.new
d.url = url
d
end
# run with examples/test-server.cr
reqs = Array.new((ARGV[0]? || 10).to_i) { request("http://127.0.0.1:8089/delay?n=#{rand(1.0)}") }
t = Time.now
ch = Channel(String).new
reqs.each do |req|
spawn do
pid, r = Process.run_with_fork do |w|
req.execute
w.puts(req.url_effective)
end
ch.send r.gets.not_nil!
r.close
end
end
reqs.size.times { puts ch.receive }
p Time.now - t