diff options
| -rw-r--r-- | .rubocop.yml | 6 | ||||
| -rw-r--r-- | lib/net/hippie.rb | 76 | ||||
| -rw-r--r-- | lib/net/hippie/client.rb | 142 | ||||
| -rw-r--r-- | lib/net/hippie/connection.rb | 82 | ||||
| -rw-r--r-- | lib/net/hippie/connection_pool.rb | 59 | ||||
| -rw-r--r-- | lib/net/hippie/dns_cache.rb | 45 | ||||
| -rw-r--r-- | lib/net/hippie/tls_parser.rb | 20 | ||||
| -rw-r--r-- | lib/net/hippie/version.rb | 2 | ||||
| -rw-r--r-- | net-hippie.gemspec | 25 | ||||
| -rw-r--r-- | test/net/client_test.rb | 2 |
10 files changed, 328 insertions, 131 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index 5e336fb..14767fc 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -1,3 +1,7 @@ +plugins: + - rubocop-minitest + - rubocop-rake + AllCops: Exclude: - 'coverage/**/*' @@ -6,7 +10,7 @@ AllCops: - 'tmp/**/*' - 'vendor/**/*' NewCops: enable - TargetRubyVersion: 3.2 + TargetRubyVersion: 4.0 Gemspec/DevelopmentDependencies: EnforcedStyle: gemspec diff --git a/lib/net/hippie.rb b/lib/net/hippie.rb index 5a9966e..a1cd91d 100644 --- a/lib/net/hippie.rb +++ b/lib/net/hippie.rb @@ -3,69 +3,83 @@ require 'base64' require 'json' require 'logger' +require 'monitor' require 'net/http' require 'openssl' +require 'resolv' +require 'timeout' -require 'net/hippie/version' require 'net/hippie/client' require 'net/hippie/connection' +require 'net/hippie/connection_pool' require 'net/hippie/content_type_mapper' +require 'net/hippie/version' module Net - # net/http for hippies. + # High-performance HTTP client with connection pooling and DNS caching. module Hippie CONNECTION_ERRORS = [ EOFError, Errno::ECONNREFUSED, Errno::ECONNRESET, - Errno::ECONNRESET, Errno::EHOSTUNREACH, Errno::EINVAL, + Errno::EPIPE, + Errno::ECONNABORTED, + Errno::ETIMEDOUT, + IOError, Net::OpenTimeout, Net::ProtocolError, Net::ReadTimeout, + Net::WriteTimeout, OpenSSL::OpenSSLError, OpenSSL::SSL::SSLError, SocketError, Timeout::Error ].freeze - def self.logger - @logger ||= Logger.new(nil) - end + BASIC_PREFIX = 'Basic ' + BEARER_PREFIX = 'Bearer ' - def self.logger=(logger) - @logger = logger - end + class << self + attr_writer :logger, :verify_mode - def self.verify_mode - @verify_mode ||= OpenSSL::SSL::VERIFY_PEER - end + def logger + @logger ||= Logger.new(nil) + end - def self.verify_mode=(mode) - @verify_mode = mode - end + def verify_mode + @verify_mode ||= OpenSSL::SSL::VERIFY_PEER + end - def self.basic_auth(username, password) - "Basic #{::Base64.strict_encode64("#{username}:#{password}")}" - end + def basic_auth(username, password) + BASIC_PREFIX + ::Base64.strict_encode64("#{username}:#{password}") + end - def self.bearer_auth(token) - "Bearer #{token}" - end + def bearer_auth(token) + BEARER_PREFIX + token + end - def self.method_missing(symbol, *args) - default_client.with_retry(retries: 3) do |client| - client.public_send(symbol, *args) - end || super - end + def default_client + @default_client ||= Client.new(follow_redirects: 3, logger: logger) + end - def self.respond_to_missing?(name, _include_private = false) - Client.public_instance_methods.include?(name.to_sym) || super - end + def reset_default_client! + @default_client = nil + end + + %i[get post put patch delete head options].each do |method| + define_method(method) do |*args, **kwargs, &block| + default_client.with_retry(retries: 3) { |c| c.public_send(method, *args, **kwargs, &block) } + end + end - def self.default_client - @default_client ||= Client.new(follow_redirects: 3, logger: logger) + def resolve(hostname, timeout: 5) + Timeout.timeout(timeout) do + addresses = Resolv.getaddresses(hostname) + addresses.find { |a| a.match?(/^\d+\.\d+\.\d+\.\d+$/) } || addresses.first + end + end end end end diff --git a/lib/net/hippie/client.rb b/lib/net/hippie/client.rb index eebb8b7..d6e072e 100644 --- a/lib/net/hippie/client.rb +++ b/lib/net/hippie/client.rb @@ -1,15 +1,22 @@ # frozen_string_literal: true +require 'net/hippie/dns_cache' +require 'net/hippie/tls_parser' + module Net module Hippie - # A simple client for connecting with http resources. + # HTTP client with connection pooling, DNS caching, and retry logic. class Client + include TlsParser + DEFAULT_HEADERS = { 'Accept' => 'application/json', 'Content-Type' => 'application/json', - 'User-Agent' => "net/hippie #{Net::Hippie::VERSION}" + 'User-Agent' => "net/hippie #{VERSION}" }.freeze + JITTER = Random.new.freeze + attr_reader :mapper, :logger, :follow_redirects def initialize(options = {}) @@ -17,102 +24,103 @@ module Net @mapper = options.fetch(:mapper, ContentTypeMapper.new) @logger = options.fetch(:logger, Net::Hippie.logger) @follow_redirects = options.fetch(:follow_redirects, 0) - @default_headers = options.fetch(:headers, DEFAULT_HEADERS) - @connections = Hash.new do |hash, key| - scheme, host, port = key - hash[key] = Connection.new(scheme, host, port, options) - end - end - - def execute(uri, request, limit: follow_redirects, &block) - connection = connection_for(uri) - return execute_with_block(connection, request, &block) if block_given? - - response = connection.run(request) - follow_redirect?(response, limit) ? follow_redirect(connection, response, limit) : response + @default_headers = options.fetch(:headers, DEFAULT_HEADERS).freeze + configure_pool(options) + configure_tls(options) end - def get(uri, headers: {}, body: {}, &block) - run(uri, Net::HTTP::Get, headers, body, &block) + %i[get post put patch delete].each do |method| + define_method(method) do |uri, headers: {}, body: {}, stream: false, &block| + run(uri, Net::HTTP.const_get(method.capitalize), headers, body, stream, &block) + end end - def patch(uri, headers: {}, body: {}, &block) - run(uri, Net::HTTP::Patch, headers, body, &block) + %i[head options].each do |method| + define_method(method) do |uri, headers: {}, &block| + run(uri, Net::HTTP.const_get(method.capitalize), headers, {}, false, &block) + end end - def post(uri, headers: {}, body: {}, &block) - run(uri, Net::HTTP::Post, headers, body, &block) - end + def execute(uri, request, limit: follow_redirects, stream: false, &block) + conn = connection_for(uri) + return conn.run(request) { |res| res.read_body(&block) } if stream && block - def put(uri, headers: {}, body: {}, &block) - run(uri, Net::HTTP::Put, headers, body, &block) - end + return execute_with_block(conn, request, &block) if block - def delete(uri, headers: {}, body: {}, &block) - run(uri, Net::HTTP::Delete, headers, body, &block) + response = conn.run(request) + limit.positive? && response.is_a?(Net::HTTPRedirection) ? follow_redirect(uri, response, limit) : response end - # attempt 1 -> delay 0.1 second - # attempt 2 -> delay 0.2 second - # attempt 3 -> delay 0.4 second - # attempt 4 -> delay 0.8 second - # attempt 5 -> delay 1.6 second - # attempt 6 -> delay 3.2 second - # attempt 7 -> delay 6.4 second - # attempt 8 -> delay 12.8 second def with_retry(retries: 3) - retries = 0 if retries.nil? || retries.negative? + retries = [retries.to_i, 0].max + 0.upto(retries) do |attempt| + return yield self + rescue *CONNECTION_ERRORS => error + raise if attempt == retries - 0.upto(retries) do |n| - attempt(n, retries) do - return yield self - end + sleep_with_backoff(attempt, retries, error) end end + def close_all + @pool.close_all + @dns_cache.clear + end + private - attr_reader :default_headers + def configure_pool(options) + @dns_ttl = options.fetch(:dns_ttl, 300) + @dns_cache = DnsCache.new( + timeout: options.fetch(:dns_timeout, 5), ttl: @dns_ttl, logger: @logger + ) + @pool = ConnectionPool.new(max_size: options.fetch(:max_connections, 100), dns_ttl: @dns_ttl) + end - def execute_with_block(connection, request, &block) - block.arity == 2 ? yield(request, connection.run(request)) : connection.run(request, &block) + def configure_tls(options) + @tls_cert = parse_cert(options[:certificate]) + @tls_key = parse_key(options[:key], options[:passphrase]) + @continue_timeout = options[:continue_timeout] + @ignore_eof = options.fetch(:ignore_eof, true) end - def follow_redirect?(response, limit) - limit.positive? && response.is_a?(Net::HTTPRedirection) + def run(uri, method, headers, body, stream, &block) + uri = URI.parse(uri.to_s) unless uri.is_a?(URI::Generic) + execute(uri, build_request(method, uri, headers, body), stream: stream, &block) end - def follow_redirect(connection, response, limit) - url = connection.build_url_for(response['location']) - request = request_for(Net::HTTP::Get, url) - execute(url, request, limit: limit - 1) + def build_request(type, uri, headers, body) + merged = headers.empty? ? @default_headers : @default_headers.merge(headers) + path = uri.respond_to?(:request_uri) ? uri.request_uri : uri.path + type.new(path, merged).tap { |req| req.body = @mapper.map_from(merged, body) unless body.empty? } end - def attempt(attempt, max) - yield - rescue *CONNECTION_ERRORS => error - raise error if attempt == max + def execute_with_block(conn, request, &block) + block.arity == 2 ? yield(request, conn.run(request)) : conn.run(request, &block) + end - delay = ((2**attempt) * 0.1) + Random.rand(0.05) # delay + jitter - logger&.warn("`#{error.message}` #{attempt + 1}/#{max} Delay: #{delay}s") - sleep delay + def connection_for(uri) + uri = URI.parse(uri.to_s) unless uri.is_a?(URI::Generic) + key = [uri.scheme, uri.host, uri.port] + @pool.checkout(key) { Connection.new(uri.scheme, uri.host, uri.port, @dns_cache.resolve(uri.host), conn_opts) } end - def request_for(type, uri, headers: {}, body: {}) - final_headers = default_headers.merge(headers) - type.new(URI.parse(uri.to_s), final_headers).tap do |x| - x.body = mapper.map_from(final_headers, body) unless body.empty? - end + def sleep_with_backoff(attempt, max_retries, error) + delay = ((2**attempt) * 0.1) + JITTER.rand(0.05) + logger&.warn("[Hippie] #{error.class}: #{error.message} | Retry #{attempt + 1}/#{max_retries}") + sleep delay end - def run(uri, http_method, headers, body, &block) - request = request_for(http_method, uri, headers: headers, body: body) - execute(uri, request, &block) + def conn_opts + @options.merge( + tls_cert: @tls_cert, tls_key: @tls_key, + continue_timeout: @continue_timeout, ignore_eof: @ignore_eof + ) end - def connection_for(uri) - uri = URI.parse(uri.to_s) - @connections[[uri.scheme, uri.host, uri.port]] + def follow_redirect(original_uri, response, limit) + redirect_uri = original_uri.merge(response['location']) + execute(redirect_uri, build_request(Net::HTTP::Get, redirect_uri, {}, {}), limit: limit - 1) end end end diff --git a/lib/net/hippie/connection.rb b/lib/net/hippie/connection.rb index 3879f10..465461f 100644 --- a/lib/net/hippie/connection.rb +++ b/lib/net/hippie/connection.rb @@ -2,27 +2,33 @@ module Net module Hippie - # A connection to a specific host + # Persistent HTTP connection with automatic reconnection. class Connection - def initialize(scheme, host, port, options = {}) - http = Net::HTTP.new(host, port) - http.read_timeout = options.fetch(:read_timeout, 10) - http.open_timeout = options.fetch(:open_timeout, 10) - http.use_ssl = scheme == 'https' - http.verify_mode = options.fetch(:verify_mode, Net::Hippie.verify_mode) - http.set_debug_output(options[:logger]) if options[:logger] - apply_client_tls_to(http, options) - @http = http + RETRYABLE_ERRORS = [EOFError, Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE, IOError].freeze + + def initialize(scheme, host, port, ipaddr, options = {}) + @mutex = Mutex.new + @created_at = Time.now + @http = build_http(scheme, host, port, ipaddr, options) end def run(request, &block) - if block_given? - @http.request(request, &block) - else - @http.request(request) + @mutex.synchronize do + ensure_started + execute(request, &block) end end + def stale?(ttl) + (Time.now - @created_at) > ttl + end + + def close + @mutex.synchronize { @http.finish if @http.started? } + rescue IOError + nil + end + def build_url_for(path) return path if path.start_with?('http') @@ -31,15 +37,51 @@ module Net private - def apply_client_tls_to(http, options) - return if options[:certificate].nil? || options[:key].nil? + def build_http(scheme, host, port, ipaddr, options) + Net::HTTP.new(host, port).tap do |http| + configure_timeouts(http, options) + configure_ssl(http, scheme, options) + configure_tls_client(http, options) + http.ipaddr = ipaddr if ipaddr + end + end - http.cert = OpenSSL::X509::Certificate.new(options[:certificate]) - http.key = private_key(options[:key], options[:passphrase]) + def configure_timeouts(http, options) + http.open_timeout = options.fetch(:open_timeout, 10) + http.read_timeout = options.fetch(:read_timeout, 10) + http.write_timeout = options.fetch(:write_timeout, 10) + http.keep_alive_timeout = options.fetch(:keep_alive_timeout, 30) + http.max_retries = options.fetch(:max_retries, 1) + http.continue_timeout = options[:continue_timeout] if options[:continue_timeout] + http.ignore_eof = options.fetch(:ignore_eof, true) + end + + def configure_ssl(http, scheme, options) + http.use_ssl = scheme == 'https' + return unless http.use_ssl? + + http.min_version = options.fetch(:min_version, :TLS1_2) + http.verify_mode = options.fetch(:verify_mode, Net::Hippie.verify_mode) + http.set_debug_output(options[:logger]) if options[:logger] + end + + def configure_tls_client(http, options) + http.cert = options[:tls_cert] if options[:tls_cert] + http.key = options[:tls_key] if options[:tls_key] end - def private_key(key, passphrase, type = OpenSSL::PKey::RSA) - passphrase ? type.new(key, passphrase) : type.new(key) + def ensure_started + @http.start unless @http.started? + end + + def execute(request, retried: false, &block) + block ? @http.request(request, &block) : @http.request(request) + rescue *RETRYABLE_ERRORS + raise if retried + + @http.finish if @http.started? + @http.start + execute(request, retried: true, &block) end end end diff --git a/lib/net/hippie/connection_pool.rb b/lib/net/hippie/connection_pool.rb new file mode 100644 index 0000000..2e220b7 --- /dev/null +++ b/lib/net/hippie/connection_pool.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Net + module Hippie + # Thread-safe connection pool with LRU eviction. + class ConnectionPool + def initialize(max_size: 100, dns_ttl: 300) + @max_size = max_size + @dns_ttl = dns_ttl + @connections = {} + @monitor = Monitor.new + end + + def checkout(key, &block) + reuse(key) || create(key, &block) + end + + def close_all + @monitor.synchronize do + @connections.each_value(&:close) + @connections.clear + end + end + + private + + def reuse(key) + @monitor.synchronize do + return nil unless @connections.key?(key) + + conn = @connections.delete(key) + return @connections[key] = conn unless conn.stale?(@dns_ttl) + + conn.close + nil + end + end + + def create(key) + conn = yield + @monitor.synchronize do + existing = reuse(key) + if existing + conn.close + return existing + end + evict_lru if @connections.size >= @max_size + @connections[key] = conn + end + end + + def evict_lru + key, conn = @connections.first + conn.close + @connections.delete(key) + end + end + end +end diff --git a/lib/net/hippie/dns_cache.rb b/lib/net/hippie/dns_cache.rb new file mode 100644 index 0000000..bbf74f8 --- /dev/null +++ b/lib/net/hippie/dns_cache.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +module Net + module Hippie + # Thread-safe DNS resolution cache with TTL. + class DnsCache + def initialize(timeout: 5, ttl: 300, logger: nil) + @timeout = timeout + @ttl = ttl + @logger = logger + @cache = {} + @monitor = Monitor.new + end + + def resolve(hostname) + cached = get(hostname) + return cached if cached + + ip = Net::Hippie.resolve(hostname, timeout: @timeout) + set(hostname, ip) + ip + rescue Timeout::Error, Resolv::ResolvError => error + @logger&.warn("[Hippie] DNS resolution failed for #{hostname}: #{error.message}") + nil + end + + def clear + @monitor.synchronize { @cache.clear } + end + + private + + def get(hostname) + @monitor.synchronize do + entry = @cache[hostname] + entry[:ip] if entry && (Time.now - entry[:time]) < @ttl + end + end + + def set(hostname, ip_addr) + @monitor.synchronize { @cache[hostname] = { ip: ip_addr, time: Time.now } } + end + end + end +end diff --git a/lib/net/hippie/tls_parser.rb b/lib/net/hippie/tls_parser.rb new file mode 100644 index 0000000..b2a9d7d --- /dev/null +++ b/lib/net/hippie/tls_parser.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Net + module Hippie + # Parses TLS certificates and keys from various formats. + module TlsParser + def parse_cert(cert) + return cert if cert.is_a?(OpenSSL::X509::Certificate) || cert.nil? + + OpenSSL::X509::Certificate.new(cert) + end + + def parse_key(key, passphrase) + return key if key.is_a?(OpenSSL::PKey::PKey) || key.nil? + + passphrase ? OpenSSL::PKey::RSA.new(key, passphrase) : OpenSSL::PKey::RSA.new(key) + end + end + end +end diff --git a/lib/net/hippie/version.rb b/lib/net/hippie/version.rb index bb07582..7e2b6ff 100644 --- a/lib/net/hippie/version.rb +++ b/lib/net/hippie/version.rb @@ -2,6 +2,6 @@ module Net module Hippie - VERSION = '1.4.0' + VERSION = '1.5.0' end end diff --git a/net-hippie.gemspec b/net-hippie.gemspec index 4fdb83e..fe25b82 100644 --- a/net-hippie.gemspec +++ b/net-hippie.gemspec @@ -12,13 +12,13 @@ Gem::Specification.new do |spec| spec.summary = 'net/http for hippies. ☮️ ' spec.description = 'net/http for hippies. ☮️ ' - spec.homepage = "https://src.mokhan.ca/xlgmokha/net-hippie" + spec.homepage = 'https://src.mokhan.ca/xlgmokha/net-hippie' spec.license = 'MIT' - spec.metadata["allowed_push_host"] = "https://rubygems.org" - spec.metadata["homepage_uri"] = spec.homepage - spec.metadata["source_code_uri"] = "https://git.mokhan.ca/xlgmokha/net-hippie.git" - spec.metadata["changelog_uri"] = "https://src.mokhan.ca/xlgmokha/net-hippie/blob/main/CHANGELOG.md.html" - + spec.metadata['allowed_push_host'] = 'https://rubygems.org' + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://git.mokhan.ca/xlgmokha/net-hippie.git' + spec.metadata['changelog_uri'] = 'https://src.mokhan.ca/xlgmokha/net-hippie/blob/main/CHANGELOG.md.html' + spec.metadata['rubygems_mfa_required'] = 'true' spec.files = `git ls-files -z`.split("\x0").reject do |f| f.match(%r{^(test|spec|features)/}) @@ -26,16 +26,21 @@ Gem::Specification.new do |spec| spec.bindir = 'exe' spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.required_ruby_version = Gem::Requirement.new('>= 3.2.0') + spec.required_ruby_version = Gem::Requirement.new('>= 4.0.0') spec.add_dependency 'base64', '~> 0.1' spec.add_dependency 'json', '~> 2.0' spec.add_dependency 'logger', '~> 1.0' - spec.add_dependency 'net-http', '~> 0.6' - spec.add_dependency 'openssl', '~> 3.0' - spec.add_development_dependency 'minitest', '~> 5.0' + spec.add_dependency 'monitor', '~> 0.1' + spec.add_dependency 'net-http', '~> 0.1' + spec.add_dependency 'openssl', '~> 4.0' + spec.add_dependency 'resolv', '~> 0.1' + spec.add_dependency 'timeout', '~> 0.1' + spec.add_development_dependency 'minitest', '~> 6.0' spec.add_development_dependency 'rake', '~> 13.0' spec.add_development_dependency 'rubocop', '~> 1.9' + spec.add_development_dependency 'rubocop-minitest', '~> 0.1' + spec.add_development_dependency 'rubocop-rake', '~> 0.1' spec.add_development_dependency 'vcr', '~> 6.0' spec.add_development_dependency 'webmock', '~> 3.4' end diff --git a/test/net/client_test.rb b/test/net/client_test.rb index 4a688d1..19e8f81 100644 --- a/test/net/client_test.rb +++ b/test/net/client_test.rb @@ -313,7 +313,7 @@ class ClientTest < Minitest::Test refute_nil response assert_kind_of Net::HTTPOK, response io.rewind - assert_match %r{^(opening connection to www.example.org:443)}, io.read + assert_match(/^opening connection to .+:443/, io.read) end end end |