Add functional tests for python Pipfile.lock

11 files changed, 407 insertions, 87 deletions

diff --git a/bin/test-all b/bin/test-all
index 44b4edc..64a5261 100755
--- a/bin/test-all
+++ b/bin/test-all
@@ -25,4 +25,5 @@ do
   QA_PROJECT=js-yarn ./bin/test
   QA_PROJECT=js-npm ./bin/test
   QA_PROJECT=csharp-nuget-dotnetcore ./bin/test
+  QA_PROJECT=python-pipenv QA_REF=pip-file-lock ./bin/test
 done
diff --git a/lib/license/management.rb b/lib/license/management.rb
index a2e0008..a6986e7 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -4,19 +4,19 @@ require 'pathname'
 require 'yaml'
 require 'json'
 require 'license_finder'
-require 'license_finder/package_managers/pipenv'
-require 'license_finder/package_utils/pypi'
 require 'license/management/loggable'
 require 'license/management/verifiable'
 require 'license/management/repository'
 require 'license/management/report'
 require 'license/management/version'
+require 'license/management/pipenv'
+require 'license/management/pypi'
 
 # This applies a monkey patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
 LicenseFinder::Scanner.const_set(
   :PACKAGE_MANAGERS,
-  LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv]
+  LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv]
 )
 
 # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786).
diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb
index a44d45d..165d8ca 100644
--- a/lib/license/management/loggable.rb
+++ b/lib/license/management/loggable.rb
@@ -10,6 +10,10 @@ module License
       def log_info(message)
         logger.info(self.class, message)
       end
+
+      def log_error(message)
+        logger.info(self.class, message, color: :red)
+      end
     end
   end
 end
diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb
new file mode 100644
index 0000000..dbab2be
--- /dev/null
+++ b/lib/license/management/pipenv.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module License
+  module Management
+    class Pipenv < LicenseFinder::PackageManager
+      include Loggable
+
+      def initialize(options = {})
+        super
+        @lockfile = Pathname('Pipfile.lock')
+      end
+
+      def current_packages
+        @current_packages ||=
+          begin
+            packages = {}
+            each_dependency(groups: allowed_groups) do |name, data, group|
+              version = canonicalize(data['version'])
+              package = packages.fetch(key_for(name, version)) do |key|
+                packages[key] = build_package_for(name, version)
+              end
+              package.groups << group
+            end
+            packages.values
+          end
+      rescue StandardError => error
+        puts error.inspect
+      end
+
+      def possible_package_paths
+        project_path ? [project_path.join(@lockfile)] : [@lockfile]
+      end
+
+      private
+
+      def each_dependency(groups: [])
+        dependencies = JSON.parse(IO.read(detected_package_path))
+        groups.each do |group|
+          dependencies[group].each do |name, data|
+            yield name, data, group
+          end
+        end
+      end
+
+      def canonicalize(version)
+        version.sub(/^==/, '')
+      end
+
+      def build_package_for(name, version)
+        LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version))
+      end
+
+      def key_for(name, version)
+        "#{name}-#{version}"
+      end
+
+      def allowed_groups
+        %w[default develop] - ignored_groups
+      end
+
+      def ignored_groups
+        @ignored_groups.to_a || []
+      end
+    end
+  end
+end
diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb
new file mode 100644
index 0000000..3c7e0ff
--- /dev/null
+++ b/lib/license/management/pypi.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'net/hippie'
+
+module License
+  module Management
+    class PyPI
+      include Loggable
+      attr_reader :http
+
+      def initialize(http = default_http)
+        @http = http
+      end
+
+      def definition_for(name, version)
+        uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json")
+        definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) })
+        log_info([name, version, definition["license"]].inspect)
+        definition
+      rescue *Net::Hippie::CONNECTION_ERRORS
+        {}
+      end
+
+      class << self
+        #def definition(name, version)
+          #@pypi ||= new
+          #@pypi.definition_for(name, version)
+        #end
+
+        def definition(name, version)
+          response = request("https://pypi.org/pypi/#{name}/#{version}/json")
+          response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
+        rescue *Net::Hippie::CONNECTION_ERRORS
+          {}
+        end
+
+        def request(location, limit = 10)
+          uri = URI(location)
+          http = Net::HTTP.new(uri.host, uri.port)
+          http.use_ssl = true
+          response = http.get(uri.request_uri).response
+          response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response
+        end
+      end
+
+      private
+
+      def process(response)
+        return JSON.parse(response.body).fetch('info', {}) if ok?(response)
+
+        log_error([response.class, response.code, response.body].inspect)
+        {}
+      end
+
+      def default_http
+        @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client|
+          client.logger = ::Logger.new('http.log')
+          client.follow_redirects = 3
+        end
+      end
+
+      def user_agent
+        "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}"
+      end
+
+      def default_headers
+        {
+          'User-Agent' => user_agent,
+          'Accept' => '*/*',
+        }
+      end
+
+      def ok?(response)
+        response.is_a?(Net::HTTPSuccess)
+      end
+    end
+  end
+end
diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb
deleted file mode 100644
index 6af53bf..0000000
--- a/lib/license_finder/package_managers/pipenv.rb
+++ /dev/null
@@ -1,60 +0,0 @@
-# frozen_string_literal: true
-
-module LicenseFinder
-  class Pipenv < PackageManager
-    def initialize(options = {})
-      super
-      @lockfile = Pathname('Pipfile.lock')
-    end
-
-    def current_packages
-      @current_packages ||=
-        begin
-          packages = {}
-          each_dependency(groups: allowed_groups) do |name, data, group|
-            version = canonicalize(data['version'])
-            package = packages.fetch(key_for(name, version)) do |key|
-              packages[key] = build_package_for(name, version)
-            end
-            package.groups << group
-          end
-          packages.values
-        end
-    end
-
-    def possible_package_paths
-      project_path ? [project_path.join(@lockfile)] : [@lockfile]
-    end
-
-    private
-
-    def each_dependency(groups: [])
-      dependencies = JSON.parse(IO.read(detected_package_path))
-      groups.each do |group|
-        dependencies[group].each do |name, data|
-          yield name, data, group
-        end
-      end
-    end
-
-    def canonicalize(version)
-      version.sub(/^==/, '')
-    end
-
-    def build_package_for(name, version)
-      PipPackage.new(name, version, PyPI.definition(name, version))
-    end
-
-    def key_for(name, version)
-      "#{name}-#{version}"
-    end
-
-    def allowed_groups
-      %w[default develop] - ignored_groups
-    end
-
-    def ignored_groups
-      @ignored_groups.to_a || []
-    end
-  end
-end
diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb
deleted file mode 100644
index e1617ee..0000000
--- a/lib/license_finder/package_utils/pypi.rb
+++ /dev/null
@@ -1,23 +0,0 @@
-# frozen_string_literal: true
-
-require 'net/hippie'
-
-module LicenseFinder
-  class PyPI
-    class << self
-      def definition(name, version)
-        url = "https://pypi.org/pypi/#{name}/#{version}/json"
-        response = http.with_retry { |client| client.get(url) }
-        response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {}
-      rescue *Net::Hippie::CONNECTION_ERRORS
-        {}
-      end
-
-      def http
-        @http ||= Net::Hippie::Client.new.tap do |client|
-          client.follow_redirects = 3
-        end
-      end
-    end
-  end
-end
diff --git a/test/results/python-pipenv-v1.1.json b/test/results/python-pipenv-v1.1.json
new file mode 100644
index 0000000..0528f88
--- /dev/null
+++ b/test/results/python-pipenv-v1.1.json
@@ -0,0 +1,102 @@
+{
+  "version": "1.1",
+  "licenses": [
+    {
+      "count": 1,
+      "name": "Apache 2.0"
+    },
+    {
+      "count": 1,
+      "name": "BSD"
+    },
+    {
+      "count": 1,
+      "name": "MIT"
+    },
+    {
+      "count": 1,
+      "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+    }
+  ],
+  "dependencies": [
+    {
+      "licenses": [
+        {
+          "name": "BSD",
+          "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29"
+        }
+      ],
+      "license": {
+        "name": "BSD",
+        "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29"
+      },
+      "dependency": {
+        "name": "django",
+        "url": "https://www.djangoproject.com/",
+        "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
+          "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)",
+          "url": ""
+        }
+      ],
+      "license": {
+        "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+      },
+      "dependency": {
+        "name": "docutils",
+        "url": "http://docutils.sourceforge.net/",
+        "description": "Docutils -- Python Documentation Utilities",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
+          "name": "MIT",
+          "url": "http://opensource.org/licenses/mit-license"
+        }
+      ],
+      "license": {
+        "name": "MIT",
+        "url": "http://opensource.org/licenses/mit-license"
+      },
+      "dependency": {
+        "name": "pytz",
+        "url": "http://pythonhosted.org/pytz",
+        "description": "World timezone definitions, modern and historical",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
+          "name": "Apache 2.0",
+          "url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
+        }
+      ],
+      "license": {
+        "name": "Apache 2.0",
+        "url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
+      },
+      "dependency": {
+        "name": "requests",
+        "url": "http://python-requests.org",
+        "description": "Python HTTP for Humans.",
+        "pathes": [
+          "."
+        ]
+      }
+    }
+  ]
+}
diff --git a/test/results/python-pipenv-v1.json b/test/results/python-pipenv-v1.json
new file mode 100644
index 0000000..6c0ae63
--- /dev/null
+++ b/test/results/python-pipenv-v1.json
@@ -0,0 +1,77 @@
+{
+  "licenses": [
+    {
+      "count": 1,
+      "name": "Apache 2.0"
+    },
+    {
+      "count": 1,
+      "name": "BSD"
+    },
+    {
+      "count": 1,
+      "name": "MIT"
+    },
+    {
+      "count": 1,
+      "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+    }
+  ],
+  "dependencies": [
+    {
+      "license": {
+        "name": "BSD",
+        "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29"
+      },
+      "dependency": {
+        "name": "django",
+        "url": "https://www.djangoproject.com/",
+        "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
+        "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)"
+      },
+      "dependency": {
+        "name": "docutils",
+        "url": "http://docutils.sourceforge.net/",
+        "description": "Docutils -- Python Documentation Utilities",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
+        "name": "MIT",
+        "url": "http://opensource.org/licenses/mit-license"
+      },
+      "dependency": {
+        "name": "pytz",
+        "url": "http://pythonhosted.org/pytz",
+        "description": "World timezone definitions, modern and historical",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
+        "name": "Apache 2.0",
+        "url": "http://www.apache.org/licenses/LICENSE-2.0.txt"
+      },
+      "dependency": {
+        "name": "requests",
+        "url": "http://python-requests.org",
+        "description": "Python HTTP for Humans.",
+        "pathes": [
+          "."
+        ]
+      }
+    }
+  ]
+}
diff --git a/test/results/python-pipenv-v2.json b/test/results/python-pipenv-v2.json
new file mode 100644
index 0000000..bdbeb14
--- /dev/null
+++ b/test/results/python-pipenv-v2.json
@@ -0,0 +1,75 @@
+{
+  "version": "2.0",
+  "licenses": [
+    {
+      "id": "Apache-2.0",
+      "name": "Apache License 2.0",
+      "url": "https://opensource.org/licenses/Apache-2.0",
+      "count": 1
+    },
+    {
+      "id": "BSD-4-Clause",
+      "name": "BSD 4-Clause \"Original\" or \"Old\" License",
+      "url": "http://directory.fsf.org/wiki/License:BSD_4Clause",
+      "count": 1
+    },
+    {
+      "id": "MIT",
+      "name": "MIT License",
+      "url": "https://opensource.org/licenses/MIT",
+      "count": 1
+    },
+    {
+      "id": "public domain, python, 2-clause bsd, gpl 3 (see copying.txt)",
+      "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)",
+      "url": "",
+      "count": 1
+    }
+  ],
+  "dependencies": [
+    {
+      "name": "django",
+      "url": "https://www.djangoproject.com/",
+      "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "BSD-4-Clause"
+      ]
+    },
+    {
+      "name": "docutils",
+      "url": "http://docutils.sourceforge.net/",
+      "description": "Docutils -- Python Documentation Utilities",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "public domain, python, 2-clause bsd, gpl 3 (see copying.txt)"
+      ]
+    },
+    {
+      "name": "pytz",
+      "url": "http://pythonhosted.org/pytz",
+      "description": "World timezone definitions, modern and historical",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "MIT"
+      ]
+    },
+    {
+      "name": "requests",
+      "url": "http://python-requests.org",
+      "description": "Python HTTP for Humans.",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "Apache-2.0"
+      ]
+    }
+  ]
+}
diff --git a/test/test.sh b/test/test.sh
index 042adc3..8d04a3a 100755
--- a/test/test.sh
+++ b/test/test.sh
@@ -22,4 +22,4 @@ mkdir -p /results/
 cp "/code/$project/gl-license-management-report.json" "/results/$project-gl-license-management-report.json"
 
 # Compare results with expected results.
-diff -u "/code/$project/gl-license-management-report.json" "/test/results/$results.json"
+diff -u "/test/results/$results.json" "/code/$project/gl-license-management-report.json"