From e98681be9b19d2a01efdbe14e7e8695b05b60010 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Mon, 13 Jan 2020 12:48:20 -0700
Subject: Inject the Pipenv package manager

---
 lib/license/management.rb                     |  7 ++++
 lib/license_finder/package_managers/pipenv.rb | 60 +++++++++++++++++++++++++++
 lib/license_finder/package_utils/pypi.rb      | 41 ++++++++++++++++++
 3 files changed, 108 insertions(+)
 create mode 100644 lib/license_finder/package_managers/pipenv.rb
 create mode 100644 lib/license_finder/package_utils/pypi.rb

(limited to 'lib')

diff --git a/lib/license/management.rb b/lib/license/management.rb
index a6e0664..a2e0008 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -2,7 +2,10 @@
 
 require 'pathname'
 require 'yaml'
+require 'json'
 require 'license_finder'
+require 'license_finder/package_managers/pipenv'
+require 'license_finder/package_utils/pypi'
 require 'license/management/loggable'
 require 'license/management/verifiable'
 require 'license/management/repository'
@@ -11,6 +14,10 @@ require 'license/management/version'
 
 # This applies a monkey patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
+LicenseFinder::Scanner.const_set(
+  :PACKAGE_MANAGERS,
+  LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv]
+)
 
 # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786).
 module LicenseFinder
diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb
new file mode 100644
index 0000000..e35b85d
--- /dev/null
+++ b/lib/license_finder/package_managers/pipenv.rb
@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module LicenseFinder
+  class Pipenv < PackageManager
+    def initialize(options = {})
+      super
+      @lockfile = Pathname('Pipfile.lock')
+    end
+
+    def current_packages
+      @current_packages ||=
+        begin
+          packages = {}
+          each_dependency(groups: allowed_groups) do |name, data, group|
+            version = canonicalize(data['version'])
+            package = packages.fetch(key_for(name, version)) do |key|
+              packages[key] = build_package_for(name, version)
+            end
+            package.groups << group
+          end
+          packages.values
+        end
+    end
+
+    def possible_package_paths
+      project_path ? [project_path.join(@lockfile)] : [@lockfile]
+    end
+
+    private
+
+    def each_dependency(groups: [])
+      dependencies = JSON.parse(IO.read(detected_package_path))
+      groups.each do |group|
+        dependencies[group].each do |name, data|
+          yield name, data, group
+        end
+      end
+    end
+
+    def canonicalize(version)
+      version.sub(/^==/, '')
+    end
+
+    def build_package_for(name, version)
+      PipPackage.new(name, version, PyPI.definition(name, version))
+    end
+
+    def key_for(name, version)
+      "#{name}-#{version}"
+    end
+
+    def allowed_groups
+      %w[default develop] - ignored_groups.to_a
+    end
+
+    def ignored_groups
+      @ignored_groups || []
+    end
+  end
+end
diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
new file mode 100644
index 0000000..fac02ec
--- /dev/null
+++ b/lib/license_finder/package_utils/pypi.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'net/http'
+require 'openssl'
+
+module LicenseFinder
+  class PyPI
+    CONNECTION_ERRORS = [
+      EOFError,
+      Errno::ECONNREFUSED,
+      Errno::ECONNRESET,
+      Errno::ECONNRESET,
+      Errno::EHOSTUNREACH,
+      Errno::EINVAL,
+      Net::OpenTimeout,
+      Net::ProtocolError,
+      Net::ReadTimeout,
+      OpenSSL::OpenSSLError,
+      OpenSSL::SSL::SSLError,
+      SocketError,
+      Timeout::Error
+    ].freeze
+
+    class << self
+      def definition(name, version)
+        response = request("https://pypi.org/pypi/#{name}/#{version}/json")
+        response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
+      rescue *CONNECTION_ERRORS
+        {}
+      end
+
+      def request(location, limit = 10)
+        uri = URI(location)
+        http = Net::HTTP.new(uri.host, uri.port)
+        http.use_ssl = true
+        response = http.get(uri.request_uri).response
+        response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
+      end
+    end
+  end
+end
-- 
cgit v1.2.3


From 8ec78f9142577f9f6ed73d4dd5ac0a3e5f02a5c6 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Mon, 13 Jan 2020 15:46:56 -0700
Subject: Use net/hippie to provider exponential backoff + jitter retries

---
 Gemfile.lock                                  |  2 ++
 lib/license_finder/package_managers/pipenv.rb |  4 ++--
 lib/license_finder/package_utils/pypi.rb      | 29 +++++----------------------
 license-management.gemspec                    |  3 ++-
 4 files changed, 11 insertions(+), 27 deletions(-)

(limited to 'lib')

diff --git a/Gemfile.lock b/Gemfile.lock
index 4978f8d..0a91a04 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -3,6 +3,7 @@ PATH
   specs:
     license-management (2.3.1)
       license_finder (~> 5.11)
+      net-hippie (~> 0.2)
 
 GEM
   remote: https://rubygems.org/
@@ -15,6 +16,7 @@ GEM
       toml (= 0.2.0)
       with_env (= 1.1.0)
       xml-simple
+    net-hippie (0.2.7)
     parslet (1.8.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb
index e35b85d..6af53bf 100644
--- a/lib/license_finder/package_managers/pipenv.rb
+++ b/lib/license_finder/package_managers/pipenv.rb
@@ -50,11 +50,11 @@ module LicenseFinder
     end
 
     def allowed_groups
-      %w[default develop] - ignored_groups.to_a
+      %w[default develop] - ignored_groups
     end
 
     def ignored_groups
-      @ignored_groups || []
+      @ignored_groups.to_a || []
     end
   end
 end
diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
index fac02ec..30856bd 100644
--- a/lib/license_finder/package_utils/pypi.rb
+++ b/lib/license_finder/package_utils/pypi.rb
@@ -1,39 +1,20 @@
 # frozen_string_literal: true
 
-require 'net/http'
-require 'openssl'
+require 'net/hippie'
 
 module LicenseFinder
   class PyPI
-    CONNECTION_ERRORS = [
-      EOFError,
-      Errno::ECONNREFUSED,
-      Errno::ECONNRESET,
-      Errno::ECONNRESET,
-      Errno::EHOSTUNREACH,
-      Errno::EINVAL,
-      Net::OpenTimeout,
-      Net::ProtocolError,
-      Net::ReadTimeout,
-      OpenSSL::OpenSSLError,
-      OpenSSL::SSL::SSLError,
-      SocketError,
-      Timeout::Error
-    ].freeze
-
     class << self
       def definition(name, version)
         response = request("https://pypi.org/pypi/#{name}/#{version}/json")
         response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
-      rescue *CONNECTION_ERRORS
+      rescue *Net::Hippie::CONNECTION_ERRORS
         {}
       end
 
-      def request(location, limit = 10)
-        uri = URI(location)
-        http = Net::HTTP.new(uri.host, uri.port)
-        http.use_ssl = true
-        response = http.get(uri.request_uri).response
+      def request(url, limit = 10)
+        client = Net::Hippie::Client.new
+        response = client.with_retry { client.get(url) }
         response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
       end
     end
diff --git a/license-management.gemspec b/license-management.gemspec
index c58bbdc..46278f0 100644
--- a/license-management.gemspec
+++ b/license-management.gemspec
@@ -13,7 +13,7 @@ Gem::Specification.new do |spec|
   spec.summary       = 'License Management job for GitLab CI.'
   spec.description   = 'License Management job for GitLab CI.'
   spec.homepage      = 'https://gitlab.com/gitlab-org/security-products/license-management'
-  spec.license       = 'GitLab EE'
+  spec.license       = 'Nonstandard'
 
   spec.metadata['allowed_push_host'] = 'https://example.com'
   spec.metadata['homepage_uri'] = spec.homepage
@@ -28,5 +28,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'license_finder', '~> 5.11'
+  spec.add_dependency 'net-hippie', '~> 0.2'
   spec.add_development_dependency 'rspec', '~> 3.9'
 end
-- 
cgit v1.2.3


From 1cc6c02f2f34252a85d354741db0a80d0f199286 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Mon, 13 Jan 2020 17:04:13 -0700
Subject: Use net/hippie to follow redirects

---
 Gemfile.lock                             |  2 +-
 lib/license_finder/package_utils/pypi.rb | 12 +++++-------
 2 files changed, 6 insertions(+), 8 deletions(-)

(limited to 'lib')

diff --git a/Gemfile.lock b/Gemfile.lock
index 0a91a04..3c700a8 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -16,7 +16,7 @@ GEM
       toml (= 0.2.0)
       with_env (= 1.1.0)
       xml-simple
-    net-hippie (0.2.7)
+    net-hippie (0.3.0)
     parslet (1.8.2)
     rspec (3.9.0)
       rspec-core (~> 3.9.0)
diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
index 30856bd..cfe3d66 100644
--- a/lib/license_finder/package_utils/pypi.rb
+++ b/lib/license_finder/package_utils/pypi.rb
@@ -6,17 +6,15 @@ module LicenseFinder
   class PyPI
     class << self
       def definition(name, version)
-        response = request("https://pypi.org/pypi/#{name}/#{version}/json")
+        url = "https://pypi.org/pypi/#{name}/#{version}/json"
+        response = Net::Hippie::Client.new.tap do |client|
+          client.follow_redirects = 3
+          client.with_retry { |x| x.get(url) }
+        end
         response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
       rescue *Net::Hippie::CONNECTION_ERRORS
         {}
       end
-
-      def request(url, limit = 10)
-        client = Net::Hippie::Client.new
-        response = client.with_retry { client.get(url) }
-        response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
-      end
     end
   end
 end
-- 
cgit v1.2.3


From 95d1ad36bde834c29f61f84cec84fe6bb91cc741 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Mon, 13 Jan 2020 17:07:35 -0700
Subject: Memoize the HTTP client

---
 lib/license_finder/package_utils/pypi.rb | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

(limited to 'lib')

diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
index cfe3d66..e1617ee 100644
--- a/lib/license_finder/package_utils/pypi.rb
+++ b/lib/license_finder/package_utils/pypi.rb
@@ -7,14 +7,17 @@ module LicenseFinder
     class << self
       def definition(name, version)
         url = "https://pypi.org/pypi/#{name}/#{version}/json"
-        response = Net::Hippie::Client.new.tap do |client|
-          client.follow_redirects = 3
-          client.with_retry { |x| x.get(url) }
-        end
+        response = http.with_retry { |client| client.get(url) }
         response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
       rescue *Net::Hippie::CONNECTION_ERRORS
         {}
       end
+
+      def http
+        @http ||= Net::Hippie::Client.new.tap do |client|
+          client.follow_redirects = 3
+        end
+      end
     end
   end
 end
-- 
cgit v1.2.3


From cfd445abb0e136471a96be27266558e488608923 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Tue, 14 Jan 2020 09:14:11 -0700
Subject: Update version and CHANGELOG

---
 CHANGELOG.md                      | 4 ++++
 CONTRIBUTING.md                   | 3 ---
 Gemfile.lock                      | 2 +-
 lib/license/management/version.rb | 2 +-
 4 files changed, 6 insertions(+), 5 deletions(-)

(limited to 'lib')

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e82432..446baa1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # GitLab License management changelog
 
+## v2.4.0
+
+- Add support for `Pipfile.lock` (!103)
+
 ## v2.3.1
 
 - Run gradle without tests by default. (!102)
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index d17d469..d8fbbde 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -31,6 +31,3 @@ open the issue in order to keep track of it and then open the relevant merge
 request that potentially fixes it.
 
 [ee-tracker]: https://gitlab.com/gitlab-org/gitlab-ee/issues
-
-
-
diff --git a/Gemfile.lock b/Gemfile.lock
index 3c700a8..b8c2990 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    license-management (2.3.1)
+    license-management (2.4.0)
       license_finder (~> 5.11)
       net-hippie (~> 0.2)
 
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
index 995dee4..a5e7b07 100644
--- a/lib/license/management/version.rb
+++ b/lib/license/management/version.rb
@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '2.3.1'
+    VERSION = '2.4.0'
   end
 end
-- 
cgit v1.2.3


From 332c5baf73ac225e820e0da47d00da3973686065 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Tue, 14 Jan 2020 13:07:07 -0700
Subject: Add functional tests for python Pipfile.lock

---
 bin/test-all                                  |   1 +
 lib/license/management.rb                     |   6 +-
 lib/license/management/loggable.rb            |   4 +
 lib/license/management/pipenv.rb              |  66 +++++++++++++++++
 lib/license/management/pypi.rb                |  78 ++++++++++++++++++++
 lib/license_finder/package_managers/pipenv.rb |  60 ---------------
 lib/license_finder/package_utils/pypi.rb      |  23 ------
 test/results/python-pipenv-v1.1.json          | 102 ++++++++++++++++++++++++++
 test/results/python-pipenv-v1.json            |  77 +++++++++++++++++++
 test/results/python-pipenv-v2.json            |  75 +++++++++++++++++++
 test/test.sh                                  |   2 +-
 11 files changed, 407 insertions(+), 87 deletions(-)
 create mode 100644 lib/license/management/pipenv.rb
 create mode 100644 lib/license/management/pypi.rb
 delete mode 100644 lib/license_finder/package_managers/pipenv.rb
 delete mode 100644 lib/license_finder/package_utils/pypi.rb
 create mode 100644 test/results/python-pipenv-v1.1.json
 create mode 100644 test/results/python-pipenv-v1.json
 create mode 100644 test/results/python-pipenv-v2.json

(limited to 'lib')

diff --git a/bin/test-all b/bin/test-all
index 44b4edc..64a5261 100755
--- a/bin/test-all
+++ b/bin/test-all
@@ -25,4 +25,5 @@ do
   QA_PROJECT=js-yarn ./bin/test
   QA_PROJECT=js-npm ./bin/test
   QA_PROJECT=csharp-nuget-dotnetcore ./bin/test
+  QA_PROJECT=python-pipenv QA_REF=pip-file-lock ./bin/test
 done
diff --git a/lib/license/management.rb b/lib/license/management.rb
index a2e0008..a6986e7 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -4,19 +4,19 @@ require 'pathname'
 require 'yaml'
 require 'json'
 require 'license_finder'
-require 'license_finder/package_managers/pipenv'
-require 'license_finder/package_utils/pypi'
 require 'license/management/loggable'
 require 'license/management/verifiable'
 require 'license/management/repository'
 require 'license/management/report'
 require 'license/management/version'
+require 'license/management/pipenv'
+require 'license/management/pypi'
 
 # This applies a monkey patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
 LicenseFinder::Scanner.const_set(
   :PACKAGE_MANAGERS,
-  LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv]
+  LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv]
 )
 
 # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786).
diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb
index a44d45d..165d8ca 100644
--- a/lib/license/management/loggable.rb
+++ b/lib/license/management/loggable.rb
@@ -10,6 +10,10 @@ module License
       def log_info(message)
         logger.info(self.class, message)
       end
+
+      def log_error(message)
+        logger.info(self.class, message, color: :red)
+      end
     end
   end
 end
diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb
new file mode 100644
index 0000000..dbab2be
--- /dev/null
+++ b/lib/license/management/pipenv.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module License
+  module Management
+    class Pipenv < LicenseFinder::PackageManager
+      include Loggable
+
+      def initialize(options = {})
+        super
+        @lockfile = Pathname('Pipfile.lock')
+      end
+
+      def current_packages
+        @current_packages ||=
+          begin
+            packages = {}
+            each_dependency(groups: allowed_groups) do |name, data, group|
+              version = canonicalize(data['version'])
+              package = packages.fetch(key_for(name, version)) do |key|
+                packages[key] = build_package_for(name, version)
+              end
+              package.groups << group
+            end
+            packages.values
+          end
+      rescue StandardError => error
+        puts error.inspect
+      end
+
+      def possible_package_paths
+        project_path ? [project_path.join(@lockfile)] : [@lockfile]
+      end
+
+      private
+
+      def each_dependency(groups: [])
+        dependencies = JSON.parse(IO.read(detected_package_path))
+        groups.each do |group|
+          dependencies[group].each do |name, data|
+            yield name, data, group
+          end
+        end
+      end
+
+      def canonicalize(version)
+        version.sub(/^==/, '')
+      end
+
+      def build_package_for(name, version)
+        LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version))
+      end
+
+      def key_for(name, version)
+        "#{name}-#{version}"
+      end
+
+      def allowed_groups
+        %w[default develop] - ignored_groups
+      end
+
+      def ignored_groups
+        @ignored_groups.to_a || []
+      end
+    end
+  end
+end
diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb
new file mode 100644
index 0000000..3c7e0ff
--- /dev/null
+++ b/lib/license/management/pypi.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'net/hippie'
+
+module License
+  module Management
+    class PyPI
+      include Loggable
+      attr_reader :http
+
+      def initialize(http = default_http)
+        @http = http
+      end
+
+      def definition_for(name, version)
+        uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json")
+        definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) })
+        log_info([name, version, definition["license"]].inspect)
+        definition
+      rescue *Net::Hippie::CONNECTION_ERRORS
+        {}
+      end
+
+      class << self
+        #def definition(name, version)
+          #@pypi ||= new
+          #@pypi.definition_for(name, version)
+        #end
+
+        def definition(name, version)
+          response = request("https://pypi.org/pypi/#{name}/#{version}/json")
+          response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
+        rescue *Net::Hippie::CONNECTION_ERRORS
+          {}
+        end
+
+        def request(location, limit = 10)
+          uri = URI(location)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          response = http.get(uri.request_uri).response
+          response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
+        end
+      end
+
+      private
+
+      def process(response)
+        return JSON.parse(response.body).fetch('info', {}) if ok?(response)
+
+        log_error([response.class, response.code, response.body].inspect)
+        {}
+      end
+
+      def default_http
+        @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client|
+          client.logger = ::Logger.new('http.log')
+          client.follow_redirects = 3
+        end
+      end
+
+      def user_agent
+        "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}"
+      end
+
+      def default_headers
+        {
+          'User-Agent' => user_agent,
+          'Accept' => '*/*',
+        }
+      end
+
+      def ok?(response)
+        response.is_a?(Net::HTTPSuccess)
+      end
+    end
+  end
+end
diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb
deleted file mode 100644
index 6af53bf..0000000
--- a/lib/license_finder/package_managers/pipenv.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module LicenseFinder
-  class Pipenv < PackageManager
-    def initialize(options = {})
-      super
-      @lockfile = Pathname('Pipfile.lock')
-    end
-
-    def current_packages
-      @current_packages ||=
-        begin
-          packages = {}
-          each_dependency(groups: allowed_groups) do |name, data, group|
-            version = canonicalize(data['version'])
-            package = packages.fetch(key_for(name, version)) do |key|
-              packages[key] = build_package_for(name, version)
-            end
-            package.groups << group
-          end
-          packages.values
-        end
-    end
-
-    def possible_package_paths
-      project_path ? [project_path.join(@lockfile)] : [@lockfile]
-    end
-
-    private
-
-    def each_dependency(groups: [])
-      dependencies = JSON.parse(IO.read(detected_package_path))
-      groups.each do |group|
-        dependencies[group].each do |name, data|
-          yield name, data, group
-        end
-      end
-    end
-
-    def canonicalize(version)
-      version.sub(/^==/, '')
-    end
-
-    def build_package_for(name, version)
-      PipPackage.new(name, version, PyPI.definition(name, version))
-    end
-
-    def key_for(name, version)
-      "#{name}-#{version}"
-    end
-
-    def allowed_groups
-      %w[default develop] - ignored_groups
-    end
-
-    def ignored_groups
-      @ignored_groups.to_a || []
-    end
-  end
-end
diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
deleted file mode 100644
index e1617ee..0000000
--- a/lib/license_finder/package_utils/pypi.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/hippie'
-
-module LicenseFinder
-  class PyPI
-    class << self
-      def definition(name, version)
-        url = "https://pypi.org/pypi/#{name}/#{version}/json"
-        response = http.with_retry { |client| client.get(url) }
-        response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
-      rescue *Net::Hippie::CONNECTION_ERRORS
-        {}
-      end
-
-      def http
-        @http ||= Net::Hippie::Client.new.tap do |client|
-          client.follow_redirects = 3
-        end
-      end
-    end
-  end
-end
diff --git a/test/results/python-pipenv-v1.1.json b/test/results/python-pipenv-v1.1.json
new file mode 100644
index 0000000..0528f88
--- /dev/null
+++ b/test/results/python-pipenv-v1.1.json
@@ -0,0 +1,102 @@
+{
+  "version": "1.1",
+  "licenses": [
+    {
+      "count": 1,
+      "name": "Apache 2.0"
+    },
+    {
+      "count": 1,
+      "name": "BSD"
+    },
+    {
+      "count": 1,
+      "name": "MIT"
+    },
+    {
+      "count": 1,
+      "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+    }
+  ],
+  "dependencies": [
+    {
+      "licenses": [
+        {
+          "name": "BSD",
+          "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29"
+        }
+      ],
+      "license": {
+        "name": "BSD",
+        "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29"
+      },
+      "dependency": {
+        "name": "django",
+        "url": "https://www.djangoproject.com/",
+        "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
+          "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)",
+          "url": ""
+        }
+      ],
+      "license": {
+        "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+      },
+      "dependency": {
+        "name": "docutils",
+        "url": "http://docutils.sourceforge.net/",
+        "description": "Docutils -- Python Documentation Utilities",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
+          "name": "MIT",
+          "url": "http://opensource.org/licenses/mit-license"
+        }
+      ],
+      "license": {
+        "name": "MIT",
+        "url": "http://opensource.org/licenses/mit-license"
+      },
+      "dependency": {
+        "name": "pytz",
+        "url": "http://pythonhosted.org/pytz",
+        "description": "World timezone definitions, modern and historical",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
+          "name": "Apache 2.0",
+          "url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
+        }
+      ],
+      "license": {
+        "name": "Apache 2.0",
+        "url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
+      },
+      "dependency": {
+        "name": "requests",
+        "url": "http://python-requests.org",
+        "description": "Python HTTP for Humans.",
+        "pathes": [
+          "."
+        ]
+      }
+    }
+  ]
+}
diff --git a/test/results/python-pipenv-v1.json b/test/results/python-pipenv-v1.json
new file mode 100644
index 0000000..6c0ae63
--- /dev/null
+++ b/test/results/python-pipenv-v1.json
@@ -0,0 +1,77 @@
+{
+  "licenses": [
+    {
+      "count": 1,
+      "name": "Apache 2.0"
+    },
+    {
+      "count": 1,
+      "name": "BSD"
+    },
+    {
+      "count": 1,
+      "name": "MIT"
+    },
+    {
+      "count": 1,
+      "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+    }
+  ],
+  "dependencies": [
+    {
+      "license": {
+        "name": "BSD",
+        "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29"
+      },
+      "dependency": {
+        "name": "django",
+        "url": "https://www.djangoproject.com/",
+        "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
+        "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+      },
+      "dependency": {
+        "name": "docutils",
+        "url": "http://docutils.sourceforge.net/",
+        "description": "Docutils -- Python Documentation Utilities",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
+        "name": "MIT",
+        "url": "http://opensource.org/licenses/mit-license"
+      },
+      "dependency": {
+        "name": "pytz",
+        "url": "http://pythonhosted.org/pytz",
+        "description": "World timezone definitions, modern and historical",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
+        "name": "Apache 2.0",
+        "url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
+      },
+      "dependency": {
+        "name": "requests",
+        "url": "http://python-requests.org",
+        "description": "Python HTTP for Humans.",
+        "pathes": [
+          "."
+        ]
+      }
+    }
+  ]
+}
diff --git a/test/results/python-pipenv-v2.json b/test/results/python-pipenv-v2.json
new file mode 100644
index 0000000..bdbeb14
--- /dev/null
+++ b/test/results/python-pipenv-v2.json
@@ -0,0 +1,75 @@
+{
+  "version": "2.0",
+  "licenses": [
+    {
+      "id": "Apache-2.0",
+      "name": "Apache License 2.0",
+      "url": "https://opensource.org/licenses/Apache-2.0",
+      "count": 1
+    },
+    {
+      "id": "BSD-4-Clause",
+      "name": "BSD 4-Clause \"Original\" or \"Old\" License",
+      "url": "http://directory.fsf.org/wiki/License:BSD_4Clause",
+      "count": 1
+    },
+    {
+      "id": "MIT",
+      "name": "MIT License",
+      "url": "https://opensource.org/licenses/MIT",
+      "count": 1
+    },
+    {
+      "id": "public domain, python, 2-clause bsd, gpl 3 (see copying.txt)",
+      "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)",
+      "url": "",
+      "count": 1
+    }
+  ],
+  "dependencies": [
+    {
+      "name": "django",
+      "url": "https://www.djangoproject.com/",
+      "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "BSD-4-Clause"
+      ]
+    },
+    {
+      "name": "docutils",
+      "url": "http://docutils.sourceforge.net/",
+      "description": "Docutils -- Python Documentation Utilities",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "public domain, python, 2-clause bsd, gpl 3 (see copying.txt)"
+      ]
+    },
+    {
+      "name": "pytz",
+      "url": "http://pythonhosted.org/pytz",
+      "description": "World timezone definitions, modern and historical",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "MIT"
+      ]
+    },
+    {
+      "name": "requests",
+      "url": "http://python-requests.org",
+      "description": "Python HTTP for Humans.",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "Apache-2.0"
+      ]
+    }
+  ]
+}
diff --git a/test/test.sh b/test/test.sh
index 042adc3..8d04a3a 100755
--- a/test/test.sh
+++ b/test/test.sh
@@ -22,4 +22,4 @@ mkdir -p /results/
 cp "/code/$project/gl-license-management-report.json" "/results/$project-gl-license-management-report.json"
 
 # Compare results with expected results.
-diff -u "/code/$project/gl-license-management-report.json" "/test/results/$results.json"
+diff -u "/test/results/$results.json" "/code/$project/gl-license-management-report.json"
-- 
cgit v1.2.3


From a2f5fb6c1ac6445fa98ee9cbb1da54e4bd3a48c6 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Tue, 14 Jan 2020 13:13:37 -0700
Subject: Remove unnecessary rescue

---
 lib/license/management/pipenv.rb | 2 --
 1 file changed, 2 deletions(-)

(limited to 'lib')

diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb
index dbab2be..482fd25 100644
--- a/lib/license/management/pipenv.rb
+++ b/lib/license/management/pipenv.rb
@@ -23,8 +23,6 @@ module License
             end
             packages.values
           end
-      rescue StandardError => error
-        puts error.inspect
       end
 
       def possible_package_paths
-- 
cgit v1.2.3


From 84e38fb1ee5feb5616a146f64f9d8f32ade5e594 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Tue, 14 Jan 2020 15:52:41 -0700
Subject: Remove unnecessary code

---
 lib/license/management/pypi.rb | 40 +++++++---------------------------------
 license-management.gemspec     |  2 +-
 2 files changed, 8 insertions(+), 34 deletions(-)

(limited to 'lib')

diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb
index 3c7e0ff..c0254e6 100644
--- a/lib/license/management/pypi.rb
+++ b/lib/license/management/pypi.rb
@@ -13,33 +13,18 @@ module License
       end
 
       def definition_for(name, version)
-        uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json")
-        definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) })
-        log_info([name, version, definition["license"]].inspect)
-        definition
+        uri = "https://pypi.org/pypi/#{name}/#{version}/json"
+        process(http.get(uri)).tap do |definition|
+          log_info([name, version, definition["license"]].inspect)
+        end
       rescue *Net::Hippie::CONNECTION_ERRORS
         {}
       end
 
       class << self
-        #def definition(name, version)
-          #@pypi ||= new
-          #@pypi.definition_for(name, version)
-        #end
-
         def definition(name, version)
-          response = request("https://pypi.org/pypi/#{name}/#{version}/json")
-          response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
-        rescue *Net::Hippie::CONNECTION_ERRORS
-          {}
-        end
-
-        def request(location, limit = 10)
-          uri = URI(location)
-          http = Net::HTTP.new(uri.host, uri.port)
-          http.use_ssl = true
-          response = http.get(uri.request_uri).response
-          response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
+          @pypi ||= new
+          @pypi.definition_for(name, version)
         end
       end
 
@@ -53,23 +38,12 @@ module License
       end
 
       def default_http
-        @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client|
+        @default_http ||= Net::Hippie::Client.new.tap do |client|
           client.logger = ::Logger.new('http.log')
           client.follow_redirects = 3
         end
       end
 
-      def user_agent
-        "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}"
-      end
-
-      def default_headers
-        {
-          'User-Agent' => user_agent,
-          'Accept' => '*/*',
-        }
-      end
-
       def ok?(response)
         response.is_a?(Net::HTTPSuccess)
       end
diff --git a/license-management.gemspec b/license-management.gemspec
index 46278f0..492fd2d 100644
--- a/license-management.gemspec
+++ b/license-management.gemspec
@@ -28,6 +28,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'license_finder', '~> 5.11'
-  spec.add_dependency 'net-hippie', '~> 0.2'
+  spec.add_dependency 'net-hippie', '~> 0.3'
   spec.add_development_dependency 'rspec', '~> 3.9'
 end
-- 
cgit v1.2.3


From ad09f31e9a846687ff692c2809f14cecd9136354 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Tue, 14 Jan 2020 16:02:55 -0700
Subject: Centralize access to HTTP to turn on airgap mode easily

---
 lib/license/management.rb      |  9 +++++++++
 lib/license/management/pypi.rb | 14 +++-----------
 2 files changed, 12 insertions(+), 11 deletions(-)

(limited to 'lib')

diff --git a/lib/license/management.rb b/lib/license/management.rb
index a6986e7..85ef4cc 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -14,6 +14,8 @@ require 'license/management/pypi'
 
 # This applies a monkey patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
+
+# This monkey patch can be removed once https://github.com/pivotal/LicenseFinder/pull/659 is released
 LicenseFinder::Scanner.const_set(
   :PACKAGE_MANAGERS,
   LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv]
@@ -38,5 +40,12 @@ module License
     def self.root
       Pathname.new(File.dirname(__FILE__)).join('../..')
     end
+
+    def self.http
+      @http ||= Net::Hippie::Client.new.tap do |client|
+        client.logger = ::Logger.new('http.log')
+        client.follow_redirects = 3
+      end
+    end
   end
 end
diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb
index c0254e6..0397532 100644
--- a/lib/license/management/pypi.rb
+++ b/lib/license/management/pypi.rb
@@ -6,15 +6,14 @@ module License
   module Management
     class PyPI
       include Loggable
-      attr_reader :http
 
-      def initialize(http = default_http)
+      def initialize(http)
         @http = http
       end
 
       def definition_for(name, version)
         uri = "https://pypi.org/pypi/#{name}/#{version}/json"
-        process(http.get(uri)).tap do |definition|
+        process(@http.with_retry { |client| client.get(uri) }).tap do |definition|
           log_info([name, version, definition["license"]].inspect)
         end
       rescue *Net::Hippie::CONNECTION_ERRORS
@@ -23,7 +22,7 @@ module License
 
       class << self
         def definition(name, version)
-          @pypi ||= new
+          @pypi ||= new(License::Management.http)
           @pypi.definition_for(name, version)
         end
       end
@@ -37,13 +36,6 @@ module License
         {}
       end
 
-      def default_http
-        @default_http ||= Net::Hippie::Client.new.tap do |client|
-          client.logger = ::Logger.new('http.log')
-          client.follow_redirects = 3
-        end
-      end
-
       def ok?(response)
         response.is_a?(Net::HTTPSuccess)
       end
-- 
cgit v1.2.3


From 1b5d49f45739e47830eaea11986e1f64880095a1 Mon Sep 17 00:00:00 2001
From: mo khan <mo.khan@gmail.com>
Date: Tue, 14 Jan 2020 16:49:42 -0700
Subject: Move files to python folder

---
 lib/license/management.rb               |  4 +--
 lib/license/management/pipenv.rb        | 64 ---------------------------------
 lib/license/management/pypi.rb          | 44 -----------------------
 lib/license/management/python/pipenv.rb | 64 +++++++++++++++++++++++++++++++++
 lib/license/management/python/pypi.rb   | 44 +++++++++++++++++++++++
 5 files changed, 110 insertions(+), 110 deletions(-)
 delete mode 100644 lib/license/management/pipenv.rb
 delete mode 100644 lib/license/management/pypi.rb
 create mode 100644 lib/license/management/python/pipenv.rb
 create mode 100644 lib/license/management/python/pypi.rb

(limited to 'lib')

diff --git a/lib/license/management.rb b/lib/license/management.rb
index 85ef4cc..ebab5e2 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -6,11 +6,11 @@ require 'json'
 require 'license_finder'
 require 'license/management/loggable'
 require 'license/management/verifiable'
+require 'license/management/python/pipenv'
+require 'license/management/python/pypi'
 require 'license/management/repository'
 require 'license/management/report'
 require 'license/management/version'
-require 'license/management/pipenv'
-require 'license/management/pypi'
 
 # This applies a monkey patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb
deleted file mode 100644
index 482fd25..0000000
--- a/lib/license/management/pipenv.rb
+++ /dev/null
@@ -1,64 +0,0 @@
-# frozen_string_literal: true
-
-module License
-  module Management
-    class Pipenv < LicenseFinder::PackageManager
-      include Loggable
-
-      def initialize(options = {})
-        super
-        @lockfile = Pathname('Pipfile.lock')
-      end
-
-      def current_packages
-        @current_packages ||=
-          begin
-            packages = {}
-            each_dependency(groups: allowed_groups) do |name, data, group|
-              version = canonicalize(data['version'])
-              package = packages.fetch(key_for(name, version)) do |key|
-                packages[key] = build_package_for(name, version)
-              end
-              package.groups << group
-            end
-            packages.values
-          end
-      end
-
-      def possible_package_paths
-        project_path ? [project_path.join(@lockfile)] : [@lockfile]
-      end
-
-      private
-
-      def each_dependency(groups: [])
-        dependencies = JSON.parse(IO.read(detected_package_path))
-        groups.each do |group|
-          dependencies[group].each do |name, data|
-            yield name, data, group
-          end
-        end
-      end
-
-      def canonicalize(version)
-        version.sub(/^==/, '')
-      end
-
-      def build_package_for(name, version)
-        LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version))
-      end
-
-      def key_for(name, version)
-        "#{name}-#{version}"
-      end
-
-      def allowed_groups
-        %w[default develop] - ignored_groups
-      end
-
-      def ignored_groups
-        @ignored_groups.to_a || []
-      end
-    end
-  end
-end
diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb
deleted file mode 100644
index 0397532..0000000
--- a/lib/license/management/pypi.rb
+++ /dev/null
@@ -1,44 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/hippie'
-
-module License
-  module Management
-    class PyPI
-      include Loggable
-
-      def initialize(http)
-        @http = http
-      end
-
-      def definition_for(name, version)
-        uri = "https://pypi.org/pypi/#{name}/#{version}/json"
-        process(@http.with_retry { |client| client.get(uri) }).tap do |definition|
-          log_info([name, version, definition["license"]].inspect)
-        end
-      rescue *Net::Hippie::CONNECTION_ERRORS
-        {}
-      end
-
-      class << self
-        def definition(name, version)
-          @pypi ||= new(License::Management.http)
-          @pypi.definition_for(name, version)
-        end
-      end
-
-      private
-
-      def process(response)
-        return JSON.parse(response.body).fetch('info', {}) if ok?(response)
-
-        log_error([response.class, response.code, response.body].inspect)
-        {}
-      end
-
-      def ok?(response)
-        response.is_a?(Net::HTTPSuccess)
-      end
-    end
-  end
-end
diff --git a/lib/license/management/python/pipenv.rb b/lib/license/management/python/pipenv.rb
new file mode 100644
index 0000000..482fd25
--- /dev/null
+++ b/lib/license/management/python/pipenv.rb
@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module License
+  module Management
+    class Pipenv < LicenseFinder::PackageManager
+      include Loggable
+
+      def initialize(options = {})
+        super
+        @lockfile = Pathname('Pipfile.lock')
+      end
+
+      def current_packages
+        @current_packages ||=
+          begin
+            packages = {}
+            each_dependency(groups: allowed_groups) do |name, data, group|
+              version = canonicalize(data['version'])
+              package = packages.fetch(key_for(name, version)) do |key|
+                packages[key] = build_package_for(name, version)
+              end
+              package.groups << group
+            end
+            packages.values
+          end
+      end
+
+      def possible_package_paths
+        project_path ? [project_path.join(@lockfile)] : [@lockfile]
+      end
+
+      private
+
+      def each_dependency(groups: [])
+        dependencies = JSON.parse(IO.read(detected_package_path))
+        groups.each do |group|
+          dependencies[group].each do |name, data|
+            yield name, data, group
+          end
+        end
+      end
+
+      def canonicalize(version)
+        version.sub(/^==/, '')
+      end
+
+      def build_package_for(name, version)
+        LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version))
+      end
+
+      def key_for(name, version)
+        "#{name}-#{version}"
+      end
+
+      def allowed_groups
+        %w[default develop] - ignored_groups
+      end
+
+      def ignored_groups
+        @ignored_groups.to_a || []
+      end
+    end
+  end
+end
diff --git a/lib/license/management/python/pypi.rb b/lib/license/management/python/pypi.rb
new file mode 100644
index 0000000..0397532
--- /dev/null
+++ b/lib/license/management/python/pypi.rb
@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require 'net/hippie'
+
+module License
+  module Management
+    class PyPI
+      include Loggable
+
+      def initialize(http)
+        @http = http
+      end
+
+      def definition_for(name, version)
+        uri = "https://pypi.org/pypi/#{name}/#{version}/json"
+        process(@http.with_retry { |client| client.get(uri) }).tap do |definition|
+          log_info([name, version, definition["license"]].inspect)
+        end
+      rescue *Net::Hippie::CONNECTION_ERRORS
+        {}
+      end
+
+      class << self
+        def definition(name, version)
+          @pypi ||= new(License::Management.http)
+          @pypi.definition_for(name, version)
+        end
+      end
+
+      private
+
+      def process(response)
+        return JSON.parse(response.body).fetch('info', {}) if ok?(response)
+
+        log_error([response.class, response.code, response.body].inspect)
+        {}
+      end
+
+      def ok?(response)
+        response.is_a?(Net::HTTPSuccess)
+      end
+    end
+  end
+end
-- 
cgit v1.2.3