Extract class to fetch data from PyPI

13 files changed, 185 insertions, 53 deletions

diff --git a/.gitignore b/.gitignore
index b04a8c8..16ed09e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,3 +9,4 @@
 
 # rspec failure tracking
 .rspec_status
+*.log
diff --git a/Gemfile.lock b/Gemfile.lock
index 1a4ec7b..2943d36 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -2,7 +2,7 @@ PATH
   remote: .
   specs:
     spandx (0.1.1)
-      net-hippie (~> 0.2)
+      net-hippie (~> 0.3)
       thor (~> 0.1)
 
 GEM
@@ -19,7 +19,7 @@ GEM
     diff-lcs (1.3)
     hashdiff (1.0.0)
     jaro_winkler (1.5.4)
-    net-hippie (0.2.7)
+    net-hippie (0.3.1)
     parallel (1.19.1)
     parser (2.7.0.0)
       ast (~> 2.4.0)
diff --git a/lib/spandx.rb b/lib/spandx.rb
index 7e1cd87..f698cf6 100644
--- a/lib/spandx.rb
+++ b/lib/spandx.rb
@@ -5,11 +5,22 @@ require 'json'
 require 'net/hippie'
 
 require 'spandx/catalogue'
-require 'spandx/catalogue_gateway'
+require 'spandx/gateways/spdx'
+require 'spandx/gateways/pypi'
 require 'spandx/license'
 require 'spandx/parsers'
 require 'spandx/version'
 
 module Spandx
   class Error < StandardError; end
+  def self.root
+    Pathname.new(File.dirname(__FILE__)).join('../..')
+  end
+
+  def self.http
+    @http ||= Net::Hippie::Client.new.tap do |client|
+      client.logger = ::Logger.new('http.log')
+      client.follow_redirects = 3
+    end
+  end
 end
diff --git a/lib/spandx/catalogue.rb b/lib/spandx/catalogue.rb
index e430c86..8871bdb 100644
--- a/lib/spandx/catalogue.rb
+++ b/lib/spandx/catalogue.rb
@@ -23,7 +23,7 @@ module Spandx
     end
 
     def self.latest
-      CatalogueGateway.new.fetch
+      ::Spandx::Gateways::Spdx.new.fetch
     end
 
     def self.from_file(path)
diff --git a/lib/spandx/catalogue_gateway.rb b/lib/spandx/catalogue_gateway.rb
deleted file mode 100644
index c730258..0000000
--- a/lib/spandx/catalogue_gateway.rb
+++ /dev/null
@@ -1,43 +0,0 @@
-# frozen_string_literal: true
-
-module Spandx
-  class CatalogueGateway
-    URL = 'https://spdx.org/licenses/licenses.json'
-
-    def initialize(http: default_client)
-      @http = http
-    end
-
-    def fetch(url: URL)
-      response = http.get(url)
-
-      if response.code == '200'
-        parse(response.body)
-      else
-        empty_catalogue
-      end
-    rescue *::Net::Hippie::CONNECTION_ERRORS
-      empty_catalogue
-    end
-
-    private
-
-    attr_reader :http
-
-    def parse(json)
-      build_catalogue(JSON.parse(json, symbolize_names: true))
-    end
-
-    def empty_catalogue
-      build_catalogue(licenses: [])
-    end
-
-    def build_catalogue(hash)
-      Catalogue.new(hash)
-    end
-
-    def default_client
-      Net::Hippie::Client.new
-    end
-  end
-end
diff --git a/lib/spandx/gateways/pypi.rb b/lib/spandx/gateways/pypi.rb
new file mode 100644
index 0000000..850d182
--- /dev/null
+++ b/lib/spandx/gateways/pypi.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Gateways
+    class PyPI
+      def initialize(http = Spandx.http)
+        @http = http
+      end
+
+      def definition_for(name, version)
+        uri = "https://pypi.org/pypi/#{name}/#{version}/json"
+        process(@http.with_retry { |client| client.get(uri) })
+      rescue *Net::Hippie::CONNECTION_ERRORS
+        {}
+      end
+
+      class << self
+        def definition(name, version)
+          @pypi ||= new
+          @pypi.definition_for(name, version)
+        end
+      end
+
+      private
+
+      def process(response)
+        return JSON.parse(response.body).fetch('info', {}) if ok?(response)
+
+        {}
+      end
+
+      def ok?(response)
+        response.is_a?(Net::HTTPSuccess)
+      end
+    end
+  end
+end
diff --git a/lib/spandx/gateways/spdx.rb b/lib/spandx/gateways/spdx.rb
new file mode 100644
index 0000000..df66546
--- /dev/null
+++ b/lib/spandx/gateways/spdx.rb
@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Gateways
+    class Spdx
+      URL = 'https://spdx.org/licenses/licenses.json'
+
+      def initialize(http: Spandx.http)
+        @http = http
+      end
+
+      def fetch(url: URL)
+        response = http.get(url)
+
+        if response.code == '200'
+          parse(response.body)
+        else
+          empty_catalogue
+        end
+      rescue *::Net::Hippie::CONNECTION_ERRORS
+        empty_catalogue
+      end
+
+      private
+
+      attr_reader :http
+
+      def parse(json)
+        build_catalogue(JSON.parse(json, symbolize_names: true))
+      end
+
+      def empty_catalogue
+        build_catalogue(licenses: [])
+      end
+
+      def build_catalogue(hash)
+        Catalogue.new(hash)
+      end
+
+      def default_client
+        Net::Hippie::Client.new
+      end
+    end
+  end
+end
diff --git a/lib/spandx/parsers/pipfile_lock.rb b/lib/spandx/parsers/pipfile_lock.rb
index 9cfd1b2..f30c2be 100644
--- a/lib/spandx/parsers/pipfile_lock.rb
+++ b/lib/spandx/parsers/pipfile_lock.rb
@@ -21,10 +21,11 @@ module Spandx
         json = JSON.parse(IO.read(lockfile), symbolize_names: true)
         json[:default].each do |key, value|
           version = value[:version].gsub(/==/, '')
+          definition = Gateways::PyPI.definition(key, version)
           yield({
             name: key,
             version: version,
-            spdx: `curl -s https://pypi.org/pypi/#{key}/#{version}/json | jq -r '.info.license'`.strip
+            spdx: definition['license']
           })
         end
       end
diff --git a/spandx.gemspec b/spandx.gemspec
index 2f02f55..0ff139f 100644
--- a/spandx.gemspec
+++ b/spandx.gemspec
@@ -28,7 +28,7 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ['lib']
 
-  spec.add_dependency 'net-hippie', '~> 0.2'
+  spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_dependency 'thor', '~> 0.1'
   spec.add_development_dependency 'bundler', '~> 2.0'
   spec.add_development_dependency 'bundler-audit', '~> 0.6'
diff --git a/spec/integration/scan_spec.rb b/spec/integration/scan_spec.rb
index 1c1c1f7..b259628 100644
--- a/spec/integration/scan_spec.rb
+++ b/spec/integration/scan_spec.rb
@@ -25,7 +25,7 @@ RSpec.describe '`spandx scan` command', type: :cli do
         "packages": [
           {
             "name": "net-hippie",
-            "version": "0.2.7",
+            "version": "0.3.1",
             "spdx": "MIT"
           }
         ]
diff --git a/spec/unit/catalogue_spec.rb b/spec/unit/catalogue_spec.rb
index deb7e4c..349f84f 100644
--- a/spec/unit/catalogue_spec.rb
+++ b/spec/unit/catalogue_spec.rb
@@ -74,11 +74,11 @@ RSpec.describe Spandx::Catalogue do
     subject { described_class.latest }
 
     context 'when the licenses.json endpoint is healthy' do
-      let(:gateway) { instance_double(Spandx::CatalogueGateway, fetch: catalogue) }
+      let(:gateway) { instance_double(Spandx::Gateways::Spdx, fetch: catalogue) }
       let(:catalogue) { instance_double(described_class) }
 
       before do
-        allow(Spandx::CatalogueGateway).to receive(:new).and_return(gateway)
+        allow(Spandx::Gateways::Spdx).to receive(:new).and_return(gateway)
       end
 
       it { expect(subject).to be(catalogue) }
diff --git a/spec/unit/gateways/pypi_spec.rb b/spec/unit/gateways/pypi_spec.rb
new file mode 100644
index 0000000..90f6689
--- /dev/null
+++ b/spec/unit/gateways/pypi_spec.rb
@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+RSpec.describe Spandx::Gateways::PyPI do
+  subject { described_class }
+
+  describe '.definition' do
+    let(:source) { 'pypi.org' }
+    let(:package) { 'six' }
+    let(:version) { '1.13.0' }
+    let(:successful_response_body) do
+      JSON.generate(
+        info: {
+          name: package,
+          version: version
+        }
+      )
+    end
+
+    context 'when the default source is reachable' do
+      before do
+        stub_request(:get, "https://#{source}/pypi/#{package}/#{version}/json")
+          .to_return(status: 200, body: successful_response_body)
+      end
+
+      specify do
+        expect(subject.definition(package, version)).to include(
+          'name' => package,
+          'version' => version
+        )
+      end
+    end
+
+    context 'when the response redirects to a different location' do
+      let(:redirect_url) { "https://#{source}/pypi/#{SecureRandom.uuid}" }
+
+      before do
+        stub_request(:get, "https://#{source}/pypi/#{package}/#{version}/json")
+          .to_return(status: 301, headers: { 'Location' => redirect_url })
+
+        stub_request(:get, redirect_url)
+          .to_return(status: 200, body: successful_response_body)
+      end
+
+      specify do
+        expect(subject.definition(package, version)).to include(
+          'name' => package,
+          'version' => version
+        )
+      end
+    end
+
+    context 'when stuck in an infinite redirect loop' do
+      before do
+        url = "https://#{source}/pypi/#{package}/#{version}/json"
+
+        11.times do |n|
+          redirect_url = "#{url}#{n}"
+          stub_request(:get, url)
+            .to_return(status: 301, headers: { 'Location' => redirect_url })
+          url = redirect_url
+        end
+      end
+
+      it 'gives up after `n` attempts' do
+        expect(subject.definition(package, version)).to be_empty
+      end
+    end
+
+    context 'when the source is not reachable' do
+      before do
+        stub_request(:get, "https://#{source}/pypi/#{package}/#{version}/json")
+          .to_timeout
+      end
+
+      it 'fails gracefully' do
+        expect(subject.definition(package, version)).to be_empty
+      end
+    end
+  end
+end
diff --git a/spec/unit/catalogue_gateway_spec.rb b/spec/unit/gateways/spdx_spec.rb
index 003ae7f..a6d9aac 100644
--- a/spec/unit/catalogue_gateway_spec.rb
+++ b/spec/unit/gateways/spdx_spec.rb
@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-RSpec.describe Spandx::CatalogueGateway do
+RSpec.describe Spandx::Gateways::Spdx do
   describe '#fetch' do
     let(:result) { subject.fetch }
     let(:url) { described_class::URL }