From 332c5baf73ac225e820e0da47d00da3973686065 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 13:07:07 -0700 Subject: Add functional tests for python Pipfile.lock --- lib/license/management.rb | 6 +-- lib/license/management/loggable.rb | 4 ++ lib/license/management/pipenv.rb | 66 +++++++++++++++++++++++ lib/license/management/pypi.rb | 78 +++++++++++++++++++++++++++ lib/license_finder/package_managers/pipenv.rb | 60 --------------------- lib/license_finder/package_utils/pypi.rb | 23 -------- 6 files changed, 151 insertions(+), 86 deletions(-) create mode 100644 lib/license/management/pipenv.rb create mode 100644 lib/license/management/pypi.rb delete mode 100644 lib/license_finder/package_managers/pipenv.rb delete mode 100644 lib/license_finder/package_utils/pypi.rb (limited to 'lib') diff --git a/lib/license/management.rb b/lib/license/management.rb index a2e0008..a6986e7 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -4,19 +4,19 @@ require 'pathname' require 'yaml' require 'json' require 'license_finder' -require 'license_finder/package_managers/pipenv' -require 'license_finder/package_utils/pypi' require 'license/management/loggable' require 'license/management/verifiable' require 'license/management/repository' require 'license/management/report' require 'license/management/version' +require 'license/management/pipenv' +require 'license/management/pypi' # This applies a monkey patch to the JsonReport found in the `license_finder` gem. LicenseFinder::JsonReport.prepend(License::Management::Report) LicenseFinder::Scanner.const_set( :PACKAGE_MANAGERS, - LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv] + LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv] ) # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786). diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb index a44d45d..165d8ca 100644 --- a/lib/license/management/loggable.rb +++ b/lib/license/management/loggable.rb @@ -10,6 +10,10 @@ module License def log_info(message) logger.info(self.class, message) end + + def log_error(message) + logger.info(self.class, message, color: :red) + end end end end diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb new file mode 100644 index 0000000..dbab2be --- /dev/null +++ b/lib/license/management/pipenv.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module License + module Management + class Pipenv < LicenseFinder::PackageManager + include Loggable + + def initialize(options = {}) + super + @lockfile = Pathname('Pipfile.lock') + end + + def current_packages + @current_packages ||= + begin + packages = {} + each_dependency(groups: allowed_groups) do |name, data, group| + version = canonicalize(data['version']) + package = packages.fetch(key_for(name, version)) do |key| + packages[key] = build_package_for(name, version) + end + package.groups << group + end + packages.values + end + rescue StandardError => error + puts error.inspect + end + + def possible_package_paths + project_path ? [project_path.join(@lockfile)] : [@lockfile] + end + + private + + def each_dependency(groups: []) + dependencies = JSON.parse(IO.read(detected_package_path)) + groups.each do |group| + dependencies[group].each do |name, data| + yield name, data, group + end + end + end + + def canonicalize(version) + version.sub(/^==/, '') + end + + def build_package_for(name, version) + LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version)) + end + + def key_for(name, version) + "#{name}-#{version}" + end + + def allowed_groups + %w[default develop] - ignored_groups + end + + def ignored_groups + @ignored_groups.to_a || [] + end + end + end +end diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb new file mode 100644 index 0000000..3c7e0ff --- /dev/null +++ b/lib/license/management/pypi.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'net/hippie' + +module License + module Management + class PyPI + include Loggable + attr_reader :http + + def initialize(http = default_http) + @http = http + end + + def definition_for(name, version) + uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json") + definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) }) + log_info([name, version, definition["license"]].inspect) + definition + rescue *Net::Hippie::CONNECTION_ERRORS + {} + end + + class << self + #def definition(name, version) + #@pypi ||= new + #@pypi.definition_for(name, version) + #end + + def definition(name, version) + response = request("https://pypi.org/pypi/#{name}/#{version}/json") + response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} + rescue *Net::Hippie::CONNECTION_ERRORS + {} + end + + def request(location, limit = 10) + uri = URI(location) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + response = http.get(uri.request_uri).response + response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response + end + end + + private + + def process(response) + return JSON.parse(response.body).fetch('info', {}) if ok?(response) + + log_error([response.class, response.code, response.body].inspect) + {} + end + + def default_http + @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client| + client.logger = ::Logger.new('http.log') + client.follow_redirects = 3 + end + end + + def user_agent + "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}" + end + + def default_headers + { + 'User-Agent' => user_agent, + 'Accept' => '*/*', + } + end + + def ok?(response) + response.is_a?(Net::HTTPSuccess) + end + end + end +end diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb deleted file mode 100644 index 6af53bf..0000000 --- a/lib/license_finder/package_managers/pipenv.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module LicenseFinder - class Pipenv < PackageManager - def initialize(options = {}) - super - @lockfile = Pathname('Pipfile.lock') - end - - def current_packages - @current_packages ||= - begin - packages = {} - each_dependency(groups: allowed_groups) do |name, data, group| - version = canonicalize(data['version']) - package = packages.fetch(key_for(name, version)) do |key| - packages[key] = build_package_for(name, version) - end - package.groups << group - end - packages.values - end - end - - def possible_package_paths - project_path ? [project_path.join(@lockfile)] : [@lockfile] - end - - private - - def each_dependency(groups: []) - dependencies = JSON.parse(IO.read(detected_package_path)) - groups.each do |group| - dependencies[group].each do |name, data| - yield name, data, group - end - end - end - - def canonicalize(version) - version.sub(/^==/, '') - end - - def build_package_for(name, version) - PipPackage.new(name, version, PyPI.definition(name, version)) - end - - def key_for(name, version) - "#{name}-#{version}" - end - - def allowed_groups - %w[default develop] - ignored_groups - end - - def ignored_groups - @ignored_groups.to_a || [] - end - end -end diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb deleted file mode 100644 index e1617ee..0000000 --- a/lib/license_finder/package_utils/pypi.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'net/hippie' - -module LicenseFinder - class PyPI - class << self - def definition(name, version) - url = "https://pypi.org/pypi/#{name}/#{version}/json" - response = http.with_retry { |client| client.get(url) } - response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} - rescue *Net::Hippie::CONNECTION_ERRORS - {} - end - - def http - @http ||= Net::Hippie::Client.new.tap do |client| - client.follow_redirects = 3 - end - end - end - end -end -- cgit v1.2.3