Add functional tests for python Pipfile.lock

6 files changed, 151 insertions, 86 deletions

diff --git a/lib/license/management.rb b/lib/license/management.rb
index a2e0008..a6986e7 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -4,19 +4,19 @@ require 'pathname'
 require 'yaml'
 require 'json'
 require 'license_finder'
-require 'license_finder/package_managers/pipenv'
-require 'license_finder/package_utils/pypi'
 require 'license/management/loggable'
 require 'license/management/verifiable'
 require 'license/management/repository'
 require 'license/management/report'
 require 'license/management/version'
+require 'license/management/pipenv'
+require 'license/management/pypi'
 
 # This applies a monkey patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
 LicenseFinder::Scanner.const_set(
   :PACKAGE_MANAGERS,
-  LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv]
+  LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv]
 )
 
 # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786).
diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb
index a44d45d..165d8ca 100644
--- a/lib/license/management/loggable.rb
+++ b/lib/license/management/loggable.rb
@@ -10,6 +10,10 @@ module License
       def log_info(message)
         logger.info(self.class, message)
       end
+
+      def log_error(message)
+        logger.info(self.class, message, color: :red)
+      end
     end
   end
 end
diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb
new file mode 100644
index 0000000..dbab2be
--- /dev/null
+++ b/lib/license/management/pipenv.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module License
+  module Management
+    class Pipenv < LicenseFinder::PackageManager
+      include Loggable
+
+      def initialize(options = {})
+        super
+        @lockfile = Pathname('Pipfile.lock')
+      end
+
+      def current_packages
+        @current_packages ||=
+          begin
+            packages = {}
+            each_dependency(groups: allowed_groups) do |name, data, group|
+              version = canonicalize(data['version'])
+              package = packages.fetch(key_for(name, version)) do |key|
+                packages[key] = build_package_for(name, version)
+              end
+              package.groups << group
+            end
+            packages.values
+          end
+      rescue StandardError => error
+        puts error.inspect
+      end
+
+      def possible_package_paths
+        project_path ? [project_path.join(@lockfile)] : [@lockfile]
+      end
+
+      private
+
+      def each_dependency(groups: [])
+        dependencies = JSON.parse(IO.read(detected_package_path))
+        groups.each do |group|
+          dependencies[group].each do |name, data|
+            yield name, data, group
+          end
+        end
+      end
+
+      def canonicalize(version)
+        version.sub(/^==/, '')
+      end
+
+      def build_package_for(name, version)
+        LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version))
+      end
+
+      def key_for(name, version)
+        "#{name}-#{version}"
+      end
+
+      def allowed_groups
+        %w[default develop] - ignored_groups
+      end
+
+      def ignored_groups
+        @ignored_groups.to_a || []
+      end
+    end
+  end
+end
diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb
new file mode 100644
index 0000000..3c7e0ff
--- /dev/null
+++ b/lib/license/management/pypi.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'net/hippie'
+
+module License
+  module Management
+    class PyPI
+      include Loggable
+      attr_reader :http
+
+      def initialize(http = default_http)
+        @http = http
+      end
+
+      def definition_for(name, version)
+        uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json")
+        definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) })
+        log_info([name, version, definition["license"]].inspect)
+        definition
+      rescue *Net::Hippie::CONNECTION_ERRORS
+        {}
+      end
+
+      class << self
+        #def definition(name, version)
+          #@pypi ||= new
+          #@pypi.definition_for(name, version)
+        #end
+
+        def definition(name, version)
+          response = request("https://pypi.org/pypi/#{name}/#{version}/json")
+          response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
+        rescue *Net::Hippie::CONNECTION_ERRORS
+          {}
+        end
+
+        def request(location, limit = 10)
+          uri = URI(location)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          response = http.get(uri.request_uri).response
+          response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
+        end
+      end
+
+      private
+
+      def process(response)
+        return JSON.parse(response.body).fetch('info', {}) if ok?(response)
+
+        log_error([response.class, response.code, response.body].inspect)
+        {}
+      end
+
+      def default_http
+        @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client|
+          client.logger = ::Logger.new('http.log')
+          client.follow_redirects = 3
+        end
+      end
+
+      def user_agent
+        "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}"
+      end
+
+      def default_headers
+        {
+          'User-Agent' => user_agent,
+          'Accept' => '*/*',
+        }
+      end
+
+      def ok?(response)
+        response.is_a?(Net::HTTPSuccess)
+      end
+    end
+  end
+end
diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb
deleted file mode 100644
index 6af53bf..0000000
--- a/lib/license_finder/package_managers/pipenv.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module LicenseFinder
-  class Pipenv < PackageManager
-    def initialize(options = {})
-      super
-      @lockfile = Pathname('Pipfile.lock')
-    end
-
-    def current_packages
-      @current_packages ||=
-        begin
-          packages = {}
-          each_dependency(groups: allowed_groups) do |name, data, group|
-            version = canonicalize(data['version'])
-            package = packages.fetch(key_for(name, version)) do |key|
-              packages[key] = build_package_for(name, version)
-            end
-            package.groups << group
-          end
-          packages.values
-        end
-    end
-
-    def possible_package_paths
-      project_path ? [project_path.join(@lockfile)] : [@lockfile]
-    end
-
-    private
-
-    def each_dependency(groups: [])
-      dependencies = JSON.parse(IO.read(detected_package_path))
-      groups.each do |group|
-        dependencies[group].each do |name, data|
-          yield name, data, group
-        end
-      end
-    end
-
-    def canonicalize(version)
-      version.sub(/^==/, '')
-    end
-
-    def build_package_for(name, version)
-      PipPackage.new(name, version, PyPI.definition(name, version))
-    end
-
-    def key_for(name, version)
-      "#{name}-#{version}"
-    end
-
-    def allowed_groups
-      %w[default develop] - ignored_groups
-    end
-
-    def ignored_groups
-      @ignored_groups.to_a || []
-    end
-  end
-end
diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
deleted file mode 100644
index e1617ee..0000000
--- a/lib/license_finder/package_utils/pypi.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/hippie'
-
-module LicenseFinder
-  class PyPI
-    class << self
-      def definition(name, version)
-        url = "https://pypi.org/pypi/#{name}/#{version}/json"
-        response = http.with_retry { |client| client.get(url) }
-        response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
-      rescue *Net::Hippie::CONNECTION_ERRORS
-        {}
-      end
-
-      def http
-        @http ||= Net::Hippie::Client.new.tap do |client|
-          client.follow_redirects = 3
-        end
-      end
-    end
-  end
-end