From a1161309d2cfe44ab1738723a6937ecd3680ea3a Mon Sep 17 00:00:00 2001 From: mo khan Date: Thu, 9 Jan 2020 09:04:48 -0700 Subject: Add spec to handle pipfile.lock files --- spec/integration/python/pipenv_spec.rb | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) create mode 100644 spec/integration/python/pipenv_spec.rb diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb new file mode 100644 index 0000000..7d02bf9 --- /dev/null +++ b/spec/integration/python/pipenv_spec.rb @@ -0,0 +1,49 @@ +require 'spec_helper' + +RSpec.describe "pipenv" do + context "when a project depends on a Pipfile.lock" do + let(:requirements) { "sentry-sdk>=0.7.7" } + + it 'produces a valid report' do + runner.add_file('Pipfile.lock') do + <<~RAW +{ + "_meta": { + "hash": { + "sha256": "d9b5cc506fc4feb9bf1ae7cadfd3737d5a0bd2b2d6c3fbcf0de3458bab34ad89" + }, + "pipfile-spec": 6, + "requires": { + "python_version": "3.8" + }, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "six": { + "hashes": [ + "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", + "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" + ], + "index": "pypi", + "version": "==1.13.0" + } + }, + "develop": {} +} + RAW + end + + report = runner.scan + + expect(report).not_to be_empty + expect(report[:version]).to start_with('2') + expect(report[:dependencies].map { |x| x[:name] }).to include("six") + end + end +end -- cgit v1.2.3 From 1c73ad594a26597086b76b4b08ef045247791b5f Mon Sep 17 00:00:00 2001 From: mo khan Date: Mon, 13 Jan 2020 11:04:04 -0700 Subject: Create tmpdir in docker reachable location for MacOS --- spec/spec_helper.rb | 1 + spec/support/integration_test_helper.rb | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index be7673c..115822b 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,5 +1,6 @@ require 'license/management' require 'json' +require 'securerandom' require 'support/integration_test_helper' RSpec.configure do |config| diff --git a/spec/support/integration_test_helper.rb b/spec/support/integration_test_helper.rb index 485af1b..5ef00a1 100644 --- a/spec/support/integration_test_helper.rb +++ b/spec/support/integration_test_helper.rb @@ -2,7 +2,8 @@ module IntegrationTestHelper class IntegrationTestRunner attr_reader :project_path - def initialize(project_path = Dir.mktmpdir('lm')) + def initialize(project_path = File.join(Dir.pwd, 'tmp', SecureRandom.uuid)) + FileUtils.mkdir_p(project_path) @project_path = project_path end -- cgit v1.2.3 From e98681be9b19d2a01efdbe14e7e8695b05b60010 Mon Sep 17 00:00:00 2001 From: mo khan Date: Mon, 13 Jan 2020 12:48:20 -0700 Subject: Inject the Pipenv package manager --- lib/license/management.rb | 7 ++++ lib/license_finder/package_managers/pipenv.rb | 60 +++++++++++++++++++++++++++ lib/license_finder/package_utils/pypi.rb | 41 ++++++++++++++++++ 3 files changed, 108 insertions(+) create mode 100644 lib/license_finder/package_managers/pipenv.rb create mode 100644 lib/license_finder/package_utils/pypi.rb diff --git a/lib/license/management.rb b/lib/license/management.rb index a6e0664..a2e0008 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -2,7 +2,10 @@ require 'pathname' require 'yaml' +require 'json' require 'license_finder' +require 'license_finder/package_managers/pipenv' +require 'license_finder/package_utils/pypi' require 'license/management/loggable' require 'license/management/verifiable' require 'license/management/repository' @@ -11,6 +14,10 @@ require 'license/management/version' # This applies a monkey patch to the JsonReport found in the `license_finder` gem. LicenseFinder::JsonReport.prepend(License::Management::Report) +LicenseFinder::Scanner.const_set( + :PACKAGE_MANAGERS, + LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv] +) # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786). module LicenseFinder diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb new file mode 100644 index 0000000..e35b85d --- /dev/null +++ b/lib/license_finder/package_managers/pipenv.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +module LicenseFinder + class Pipenv < PackageManager + def initialize(options = {}) + super + @lockfile = Pathname('Pipfile.lock') + end + + def current_packages + @current_packages ||= + begin + packages = {} + each_dependency(groups: allowed_groups) do |name, data, group| + version = canonicalize(data['version']) + package = packages.fetch(key_for(name, version)) do |key| + packages[key] = build_package_for(name, version) + end + package.groups << group + end + packages.values + end + end + + def possible_package_paths + project_path ? [project_path.join(@lockfile)] : [@lockfile] + end + + private + + def each_dependency(groups: []) + dependencies = JSON.parse(IO.read(detected_package_path)) + groups.each do |group| + dependencies[group].each do |name, data| + yield name, data, group + end + end + end + + def canonicalize(version) + version.sub(/^==/, '') + end + + def build_package_for(name, version) + PipPackage.new(name, version, PyPI.definition(name, version)) + end + + def key_for(name, version) + "#{name}-#{version}" + end + + def allowed_groups + %w[default develop] - ignored_groups.to_a + end + + def ignored_groups + @ignored_groups || [] + end + end +end diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb new file mode 100644 index 0000000..fac02ec --- /dev/null +++ b/lib/license_finder/package_utils/pypi.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'net/http' +require 'openssl' + +module LicenseFinder + class PyPI + CONNECTION_ERRORS = [ + EOFError, + Errno::ECONNREFUSED, + Errno::ECONNRESET, + Errno::ECONNRESET, + Errno::EHOSTUNREACH, + Errno::EINVAL, + Net::OpenTimeout, + Net::ProtocolError, + Net::ReadTimeout, + OpenSSL::OpenSSLError, + OpenSSL::SSL::SSLError, + SocketError, + Timeout::Error + ].freeze + + class << self + def definition(name, version) + response = request("https://pypi.org/pypi/#{name}/#{version}/json") + response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} + rescue *CONNECTION_ERRORS + {} + end + + def request(location, limit = 10) + uri = URI(location) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + response = http.get(uri.request_uri).response + response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response + end + end + end +end -- cgit v1.2.3 From 8ec78f9142577f9f6ed73d4dd5ac0a3e5f02a5c6 Mon Sep 17 00:00:00 2001 From: mo khan Date: Mon, 13 Jan 2020 15:46:56 -0700 Subject: Use net/hippie to provider exponential backoff + jitter retries --- Gemfile.lock | 2 ++ lib/license_finder/package_managers/pipenv.rb | 4 ++-- lib/license_finder/package_utils/pypi.rb | 29 +++++---------------------- license-management.gemspec | 3 ++- 4 files changed, 11 insertions(+), 27 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 4978f8d..0a91a04 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,6 +3,7 @@ PATH specs: license-management (2.3.1) license_finder (~> 5.11) + net-hippie (~> 0.2) GEM remote: https://rubygems.org/ @@ -15,6 +16,7 @@ GEM toml (= 0.2.0) with_env (= 1.1.0) xml-simple + net-hippie (0.2.7) parslet (1.8.2) rspec (3.9.0) rspec-core (~> 3.9.0) diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb index e35b85d..6af53bf 100644 --- a/lib/license_finder/package_managers/pipenv.rb +++ b/lib/license_finder/package_managers/pipenv.rb @@ -50,11 +50,11 @@ module LicenseFinder end def allowed_groups - %w[default develop] - ignored_groups.to_a + %w[default develop] - ignored_groups end def ignored_groups - @ignored_groups || [] + @ignored_groups.to_a || [] end end end diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb index fac02ec..30856bd 100644 --- a/lib/license_finder/package_utils/pypi.rb +++ b/lib/license_finder/package_utils/pypi.rb @@ -1,39 +1,20 @@ # frozen_string_literal: true -require 'net/http' -require 'openssl' +require 'net/hippie' module LicenseFinder class PyPI - CONNECTION_ERRORS = [ - EOFError, - Errno::ECONNREFUSED, - Errno::ECONNRESET, - Errno::ECONNRESET, - Errno::EHOSTUNREACH, - Errno::EINVAL, - Net::OpenTimeout, - Net::ProtocolError, - Net::ReadTimeout, - OpenSSL::OpenSSLError, - OpenSSL::SSL::SSLError, - SocketError, - Timeout::Error - ].freeze - class << self def definition(name, version) response = request("https://pypi.org/pypi/#{name}/#{version}/json") response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} - rescue *CONNECTION_ERRORS + rescue *Net::Hippie::CONNECTION_ERRORS {} end - def request(location, limit = 10) - uri = URI(location) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true - response = http.get(uri.request_uri).response + def request(url, limit = 10) + client = Net::Hippie::Client.new + response = client.with_retry { client.get(url) } response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response end end diff --git a/license-management.gemspec b/license-management.gemspec index c58bbdc..46278f0 100644 --- a/license-management.gemspec +++ b/license-management.gemspec @@ -13,7 +13,7 @@ Gem::Specification.new do |spec| spec.summary = 'License Management job for GitLab CI.' spec.description = 'License Management job for GitLab CI.' spec.homepage = 'https://gitlab.com/gitlab-org/security-products/license-management' - spec.license = 'GitLab EE' + spec.license = 'Nonstandard' spec.metadata['allowed_push_host'] = 'https://example.com' spec.metadata['homepage_uri'] = spec.homepage @@ -28,5 +28,6 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'license_finder', '~> 5.11' + spec.add_dependency 'net-hippie', '~> 0.2' spec.add_development_dependency 'rspec', '~> 3.9' end -- cgit v1.2.3 From 1cc6c02f2f34252a85d354741db0a80d0f199286 Mon Sep 17 00:00:00 2001 From: mo khan Date: Mon, 13 Jan 2020 17:04:13 -0700 Subject: Use net/hippie to follow redirects --- Gemfile.lock | 2 +- lib/license_finder/package_utils/pypi.rb | 12 +++++------- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/Gemfile.lock b/Gemfile.lock index 0a91a04..3c700a8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -16,7 +16,7 @@ GEM toml (= 0.2.0) with_env (= 1.1.0) xml-simple - net-hippie (0.2.7) + net-hippie (0.3.0) parslet (1.8.2) rspec (3.9.0) rspec-core (~> 3.9.0) diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb index 30856bd..cfe3d66 100644 --- a/lib/license_finder/package_utils/pypi.rb +++ b/lib/license_finder/package_utils/pypi.rb @@ -6,17 +6,15 @@ module LicenseFinder class PyPI class << self def definition(name, version) - response = request("https://pypi.org/pypi/#{name}/#{version}/json") + url = "https://pypi.org/pypi/#{name}/#{version}/json" + response = Net::Hippie::Client.new.tap do |client| + client.follow_redirects = 3 + client.with_retry { |x| x.get(url) } + end response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} rescue *Net::Hippie::CONNECTION_ERRORS {} end - - def request(url, limit = 10) - client = Net::Hippie::Client.new - response = client.with_retry { client.get(url) } - response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response - end end end end -- cgit v1.2.3 From 95d1ad36bde834c29f61f84cec84fe6bb91cc741 Mon Sep 17 00:00:00 2001 From: mo khan Date: Mon, 13 Jan 2020 17:07:35 -0700 Subject: Memoize the HTTP client --- lib/license_finder/package_utils/pypi.rb | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb index cfe3d66..e1617ee 100644 --- a/lib/license_finder/package_utils/pypi.rb +++ b/lib/license_finder/package_utils/pypi.rb @@ -7,14 +7,17 @@ module LicenseFinder class << self def definition(name, version) url = "https://pypi.org/pypi/#{name}/#{version}/json" - response = Net::Hippie::Client.new.tap do |client| - client.follow_redirects = 3 - client.with_retry { |x| x.get(url) } - end + response = http.with_retry { |client| client.get(url) } response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} rescue *Net::Hippie::CONNECTION_ERRORS {} end + + def http + @http ||= Net::Hippie::Client.new.tap do |client| + client.follow_redirects = 3 + end + end end end end -- cgit v1.2.3 From cfd445abb0e136471a96be27266558e488608923 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 09:14:11 -0700 Subject: Update version and CHANGELOG --- CHANGELOG.md | 4 ++++ CONTRIBUTING.md | 3 --- Gemfile.lock | 2 +- lib/license/management/version.rb | 2 +- 4 files changed, 6 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e82432..446baa1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # GitLab License management changelog +## v2.4.0 + +- Add support for `Pipfile.lock` (!103) + ## v2.3.1 - Run gradle without tests by default. (!102) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d17d469..d8fbbde 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -31,6 +31,3 @@ open the issue in order to keep track of it and then open the relevant merge request that potentially fixes it. [ee-tracker]: https://gitlab.com/gitlab-org/gitlab-ee/issues - - - diff --git a/Gemfile.lock b/Gemfile.lock index 3c700a8..b8c2990 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - license-management (2.3.1) + license-management (2.4.0) license_finder (~> 5.11) net-hippie (~> 0.2) diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index 995dee4..a5e7b07 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '2.3.1' + VERSION = '2.4.0' end end -- cgit v1.2.3 From 332c5baf73ac225e820e0da47d00da3973686065 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 13:07:07 -0700 Subject: Add functional tests for python Pipfile.lock --- bin/test-all | 1 + lib/license/management.rb | 6 +- lib/license/management/loggable.rb | 4 + lib/license/management/pipenv.rb | 66 +++++++++++++++++ lib/license/management/pypi.rb | 78 ++++++++++++++++++++ lib/license_finder/package_managers/pipenv.rb | 60 --------------- lib/license_finder/package_utils/pypi.rb | 23 ------ test/results/python-pipenv-v1.1.json | 102 ++++++++++++++++++++++++++ test/results/python-pipenv-v1.json | 77 +++++++++++++++++++ test/results/python-pipenv-v2.json | 75 +++++++++++++++++++ test/test.sh | 2 +- 11 files changed, 407 insertions(+), 87 deletions(-) create mode 100644 lib/license/management/pipenv.rb create mode 100644 lib/license/management/pypi.rb delete mode 100644 lib/license_finder/package_managers/pipenv.rb delete mode 100644 lib/license_finder/package_utils/pypi.rb create mode 100644 test/results/python-pipenv-v1.1.json create mode 100644 test/results/python-pipenv-v1.json create mode 100644 test/results/python-pipenv-v2.json diff --git a/bin/test-all b/bin/test-all index 44b4edc..64a5261 100755 --- a/bin/test-all +++ b/bin/test-all @@ -25,4 +25,5 @@ do QA_PROJECT=js-yarn ./bin/test QA_PROJECT=js-npm ./bin/test QA_PROJECT=csharp-nuget-dotnetcore ./bin/test + QA_PROJECT=python-pipenv QA_REF=pip-file-lock ./bin/test done diff --git a/lib/license/management.rb b/lib/license/management.rb index a2e0008..a6986e7 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -4,19 +4,19 @@ require 'pathname' require 'yaml' require 'json' require 'license_finder' -require 'license_finder/package_managers/pipenv' -require 'license_finder/package_utils/pypi' require 'license/management/loggable' require 'license/management/verifiable' require 'license/management/repository' require 'license/management/report' require 'license/management/version' +require 'license/management/pipenv' +require 'license/management/pypi' # This applies a monkey patch to the JsonReport found in the `license_finder` gem. LicenseFinder::JsonReport.prepend(License::Management::Report) LicenseFinder::Scanner.const_set( :PACKAGE_MANAGERS, - LicenseFinder::Scanner::PACKAGE_MANAGERS + [LicenseFinder::Pipenv] + LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv] ) # This monkey patch can be removed once we upgrade to license_finder 5.9.2. Details [here](https://gitlab.com/gitlab-org/gitlab/issues/13748#note_235810786). diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb index a44d45d..165d8ca 100644 --- a/lib/license/management/loggable.rb +++ b/lib/license/management/loggable.rb @@ -10,6 +10,10 @@ module License def log_info(message) logger.info(self.class, message) end + + def log_error(message) + logger.info(self.class, message, color: :red) + end end end end diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb new file mode 100644 index 0000000..dbab2be --- /dev/null +++ b/lib/license/management/pipenv.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module License + module Management + class Pipenv < LicenseFinder::PackageManager + include Loggable + + def initialize(options = {}) + super + @lockfile = Pathname('Pipfile.lock') + end + + def current_packages + @current_packages ||= + begin + packages = {} + each_dependency(groups: allowed_groups) do |name, data, group| + version = canonicalize(data['version']) + package = packages.fetch(key_for(name, version)) do |key| + packages[key] = build_package_for(name, version) + end + package.groups << group + end + packages.values + end + rescue StandardError => error + puts error.inspect + end + + def possible_package_paths + project_path ? [project_path.join(@lockfile)] : [@lockfile] + end + + private + + def each_dependency(groups: []) + dependencies = JSON.parse(IO.read(detected_package_path)) + groups.each do |group| + dependencies[group].each do |name, data| + yield name, data, group + end + end + end + + def canonicalize(version) + version.sub(/^==/, '') + end + + def build_package_for(name, version) + LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version)) + end + + def key_for(name, version) + "#{name}-#{version}" + end + + def allowed_groups + %w[default develop] - ignored_groups + end + + def ignored_groups + @ignored_groups.to_a || [] + end + end + end +end diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb new file mode 100644 index 0000000..3c7e0ff --- /dev/null +++ b/lib/license/management/pypi.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +require 'net/hippie' + +module License + module Management + class PyPI + include Loggable + attr_reader :http + + def initialize(http = default_http) + @http = http + end + + def definition_for(name, version) + uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json") + definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) }) + log_info([name, version, definition["license"]].inspect) + definition + rescue *Net::Hippie::CONNECTION_ERRORS + {} + end + + class << self + #def definition(name, version) + #@pypi ||= new + #@pypi.definition_for(name, version) + #end + + def definition(name, version) + response = request("https://pypi.org/pypi/#{name}/#{version}/json") + response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} + rescue *Net::Hippie::CONNECTION_ERRORS + {} + end + + def request(location, limit = 10) + uri = URI(location) + http = Net::HTTP.new(uri.host, uri.port) + http.use_ssl = true + response = http.get(uri.request_uri).response + response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response + end + end + + private + + def process(response) + return JSON.parse(response.body).fetch('info', {}) if ok?(response) + + log_error([response.class, response.code, response.body].inspect) + {} + end + + def default_http + @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client| + client.logger = ::Logger.new('http.log') + client.follow_redirects = 3 + end + end + + def user_agent + "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}" + end + + def default_headers + { + 'User-Agent' => user_agent, + 'Accept' => '*/*', + } + end + + def ok?(response) + response.is_a?(Net::HTTPSuccess) + end + end + end +end diff --git a/lib/license_finder/package_managers/pipenv.rb b/lib/license_finder/package_managers/pipenv.rb deleted file mode 100644 index 6af53bf..0000000 --- a/lib/license_finder/package_managers/pipenv.rb +++ /dev/null @@ -1,60 +0,0 @@ -# frozen_string_literal: true - -module LicenseFinder - class Pipenv < PackageManager - def initialize(options = {}) - super - @lockfile = Pathname('Pipfile.lock') - end - - def current_packages - @current_packages ||= - begin - packages = {} - each_dependency(groups: allowed_groups) do |name, data, group| - version = canonicalize(data['version']) - package = packages.fetch(key_for(name, version)) do |key| - packages[key] = build_package_for(name, version) - end - package.groups << group - end - packages.values - end - end - - def possible_package_paths - project_path ? [project_path.join(@lockfile)] : [@lockfile] - end - - private - - def each_dependency(groups: []) - dependencies = JSON.parse(IO.read(detected_package_path)) - groups.each do |group| - dependencies[group].each do |name, data| - yield name, data, group - end - end - end - - def canonicalize(version) - version.sub(/^==/, '') - end - - def build_package_for(name, version) - PipPackage.new(name, version, PyPI.definition(name, version)) - end - - def key_for(name, version) - "#{name}-#{version}" - end - - def allowed_groups - %w[default develop] - ignored_groups - end - - def ignored_groups - @ignored_groups.to_a || [] - end - end -end diff --git a/lib/license_finder/package_utils/pypi.rb b/lib/license_finder/package_utils/pypi.rb deleted file mode 100644 index e1617ee..0000000 --- a/lib/license_finder/package_utils/pypi.rb +++ /dev/null @@ -1,23 +0,0 @@ -# frozen_string_literal: true - -require 'net/hippie' - -module LicenseFinder - class PyPI - class << self - def definition(name, version) - url = "https://pypi.org/pypi/#{name}/#{version}/json" - response = http.with_retry { |client| client.get(url) } - response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} - rescue *Net::Hippie::CONNECTION_ERRORS - {} - end - - def http - @http ||= Net::Hippie::Client.new.tap do |client| - client.follow_redirects = 3 - end - end - end - end -end diff --git a/test/results/python-pipenv-v1.1.json b/test/results/python-pipenv-v1.1.json new file mode 100644 index 0000000..0528f88 --- /dev/null +++ b/test/results/python-pipenv-v1.1.json @@ -0,0 +1,102 @@ +{ + "version": "1.1", + "licenses": [ + { + "count": 1, + "name": "Apache 2.0" + }, + { + "count": 1, + "name": "BSD" + }, + { + "count": 1, + "name": "MIT" + }, + { + "count": 1, + "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)" + } + ], + "dependencies": [ + { + "licenses": [ + { + "name": "BSD", + "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29" + } + ], + "license": { + "name": "BSD", + "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29" + }, + "dependency": { + "name": "django", + "url": "https://www.djangoproject.com/", + "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.", + "pathes": [ + "." + ] + } + }, + { + "licenses": [ + { + "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)", + "url": "" + } + ], + "license": { + "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)" + }, + "dependency": { + "name": "docutils", + "url": "http://docutils.sourceforge.net/", + "description": "Docutils -- Python Documentation Utilities", + "pathes": [ + "." + ] + } + }, + { + "licenses": [ + { + "name": "MIT", + "url": "http://opensource.org/licenses/mit-license" + } + ], + "license": { + "name": "MIT", + "url": "http://opensource.org/licenses/mit-license" + }, + "dependency": { + "name": "pytz", + "url": "http://pythonhosted.org/pytz", + "description": "World timezone definitions, modern and historical", + "pathes": [ + "." + ] + } + }, + { + "licenses": [ + { + "name": "Apache 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + } + ], + "license": { + "name": "Apache 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, + "dependency": { + "name": "requests", + "url": "http://python-requests.org", + "description": "Python HTTP for Humans.", + "pathes": [ + "." + ] + } + } + ] +} diff --git a/test/results/python-pipenv-v1.json b/test/results/python-pipenv-v1.json new file mode 100644 index 0000000..6c0ae63 --- /dev/null +++ b/test/results/python-pipenv-v1.json @@ -0,0 +1,77 @@ +{ + "licenses": [ + { + "count": 1, + "name": "Apache 2.0" + }, + { + "count": 1, + "name": "BSD" + }, + { + "count": 1, + "name": "MIT" + }, + { + "count": 1, + "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)" + } + ], + "dependencies": [ + { + "license": { + "name": "BSD", + "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29" + }, + "dependency": { + "name": "django", + "url": "https://www.djangoproject.com/", + "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.", + "pathes": [ + "." + ] + } + }, + { + "license": { + "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)" + }, + "dependency": { + "name": "docutils", + "url": "http://docutils.sourceforge.net/", + "description": "Docutils -- Python Documentation Utilities", + "pathes": [ + "." + ] + } + }, + { + "license": { + "name": "MIT", + "url": "http://opensource.org/licenses/mit-license" + }, + "dependency": { + "name": "pytz", + "url": "http://pythonhosted.org/pytz", + "description": "World timezone definitions, modern and historical", + "pathes": [ + "." + ] + } + }, + { + "license": { + "name": "Apache 2.0", + "url": "http://www.apache.org/licenses/LICENSE-2.0.txt" + }, + "dependency": { + "name": "requests", + "url": "http://python-requests.org", + "description": "Python HTTP for Humans.", + "pathes": [ + "." + ] + } + } + ] +} diff --git a/test/results/python-pipenv-v2.json b/test/results/python-pipenv-v2.json new file mode 100644 index 0000000..bdbeb14 --- /dev/null +++ b/test/results/python-pipenv-v2.json @@ -0,0 +1,75 @@ +{ + "version": "2.0", + "licenses": [ + { + "id": "Apache-2.0", + "name": "Apache License 2.0", + "url": "https://opensource.org/licenses/Apache-2.0", + "count": 1 + }, + { + "id": "BSD-4-Clause", + "name": "BSD 4-Clause \"Original\" or \"Old\" License", + "url": "http://directory.fsf.org/wiki/License:BSD_4Clause", + "count": 1 + }, + { + "id": "MIT", + "name": "MIT License", + "url": "https://opensource.org/licenses/MIT", + "count": 1 + }, + { + "id": "public domain, python, 2-clause bsd, gpl 3 (see copying.txt)", + "name": "public domain, Python, 2-Clause BSD, GPL 3 (see COPYING.txt)", + "url": "", + "count": 1 + } + ], + "dependencies": [ + { + "name": "django", + "url": "https://www.djangoproject.com/", + "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.", + "paths": [ + "." + ], + "licenses": [ + "BSD-4-Clause" + ] + }, + { + "name": "docutils", + "url": "http://docutils.sourceforge.net/", + "description": "Docutils -- Python Documentation Utilities", + "paths": [ + "." + ], + "licenses": [ + "public domain, python, 2-clause bsd, gpl 3 (see copying.txt)" + ] + }, + { + "name": "pytz", + "url": "http://pythonhosted.org/pytz", + "description": "World timezone definitions, modern and historical", + "paths": [ + "." + ], + "licenses": [ + "MIT" + ] + }, + { + "name": "requests", + "url": "http://python-requests.org", + "description": "Python HTTP for Humans.", + "paths": [ + "." + ], + "licenses": [ + "Apache-2.0" + ] + } + ] +} diff --git a/test/test.sh b/test/test.sh index 042adc3..8d04a3a 100755 --- a/test/test.sh +++ b/test/test.sh @@ -22,4 +22,4 @@ mkdir -p /results/ cp "/code/$project/gl-license-management-report.json" "/results/$project-gl-license-management-report.json" # Compare results with expected results. -diff -u "/code/$project/gl-license-management-report.json" "/test/results/$results.json" +diff -u "/test/results/$results.json" "/code/$project/gl-license-management-report.json" -- cgit v1.2.3 From 0d2ada02763b14f288bbf848749d31fc17d2c59a Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 13:09:02 -0700 Subject: Add gitlab ci jobs for python-pipenv --- .gitlab-ci.yml | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 02c188c..9f4f6de 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -145,6 +145,25 @@ QA:python3-pip-v2: LM_REPORT_VERSION: 2 QA_RESULTS: python3-pip-v2 +QA:python3-pipenv: + extends: .QA + variables: + QA_RESULTS: python3-pipenv-v1 + QA_PROJECT: python-pipenv + QA_REF: pip-file-lock + +QA:python3-pipenv-v1-1: + extends: QA:python3-pip + variables: + LM_REPORT_VERSION: '1.1' + QA_RESULTS: python3-pipenv-v1.1 + +QA:python3-pipenv-v2: + extends: QA:python3-pipenv + variables: + LM_REPORT_VERSION: 2 + QA_RESULTS: python3-pipenv-v2 + QA:ruby-bundler: extends: .QA variables: -- cgit v1.2.3 From a2f5fb6c1ac6445fa98ee9cbb1da54e4bd3a48c6 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 13:13:37 -0700 Subject: Remove unnecessary rescue --- lib/license/management/pipenv.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb index dbab2be..482fd25 100644 --- a/lib/license/management/pipenv.rb +++ b/lib/license/management/pipenv.rb @@ -23,8 +23,6 @@ module License end packages.values end - rescue StandardError => error - puts error.inspect end def possible_package_paths -- cgit v1.2.3 From eb3a7788d67b05faaa73bb521635251ae14e7d2a Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 13:16:47 -0700 Subject: Simplify fixture generation in test --- spec/integration/python/pipenv_spec.rb | 49 +++++++++++----------------------- 1 file changed, 15 insertions(+), 34 deletions(-) diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb index 7d02bf9..15e48dc 100644 --- a/spec/integration/python/pipenv_spec.rb +++ b/spec/integration/python/pipenv_spec.rb @@ -2,42 +2,23 @@ require 'spec_helper' RSpec.describe "pipenv" do context "when a project depends on a Pipfile.lock" do - let(:requirements) { "sentry-sdk>=0.7.7" } - - it 'produces a valid report' do - runner.add_file('Pipfile.lock') do - <<~RAW -{ - "_meta": { - "hash": { - "sha256": "d9b5cc506fc4feb9bf1ae7cadfd3737d5a0bd2b2d6c3fbcf0de3458bab34ad89" + let(:pipfile_lock_content) do + JSON.pretty_generate({ + "_meta": { + "hash": { "sha256": "" }, + "pipfile-spec": 6, + "requires": { "python_version": "3.8" }, + "sources": [ { "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true } ] }, - "pipfile-spec": 6, - "requires": { - "python_version": "3.8" + "default": { + "six": { "hashes": [], "index": "pypi", "version": "==1.13.0" } }, - "sources": [ - { - "name": "pypi", - "url": "https://pypi.org/simple", - "verify_ssl": true - } - ] - }, - "default": { - "six": { - "hashes": [ - "sha256:1f1b7d42e254082a9db6279deae68afb421ceba6158efa6131de7b3003ee93fd", - "sha256:30f610279e8b2578cab6db20741130331735c781b56053c59c4076da27f06b66" - ], - "index": "pypi", - "version": "==1.13.0" - } - }, - "develop": {} -} - RAW - end + "develop": {} + }) + end + + it 'produces a valid report' do + runner.add_file('Pipfile.lock', pipfile_lock_content) report = runner.scan -- cgit v1.2.3 From 2812c94c95f735e9956dcacf2599cd080cb956b8 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 14:43:18 -0700 Subject: Fix name of text fixture file --- .gitlab-ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9f4f6de..0a29b68 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -148,7 +148,7 @@ QA:python3-pip-v2: QA:python3-pipenv: extends: .QA variables: - QA_RESULTS: python3-pipenv-v1 + QA_RESULTS: python-pipenv-v1 QA_PROJECT: python-pipenv QA_REF: pip-file-lock @@ -156,13 +156,13 @@ QA:python3-pipenv-v1-1: extends: QA:python3-pip variables: LM_REPORT_VERSION: '1.1' - QA_RESULTS: python3-pipenv-v1.1 + QA_RESULTS: python-pipenv-v1.1 QA:python3-pipenv-v2: extends: QA:python3-pipenv variables: LM_REPORT_VERSION: 2 - QA_RESULTS: python3-pipenv-v2 + QA_RESULTS: python-pipenv-v2 QA:ruby-bundler: extends: .QA -- cgit v1.2.3 From 84e38fb1ee5feb5616a146f64f9d8f32ade5e594 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 15:52:41 -0700 Subject: Remove unnecessary code --- lib/license/management/pypi.rb | 40 +++++++--------------------------------- license-management.gemspec | 2 +- 2 files changed, 8 insertions(+), 34 deletions(-) diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb index 3c7e0ff..c0254e6 100644 --- a/lib/license/management/pypi.rb +++ b/lib/license/management/pypi.rb @@ -13,33 +13,18 @@ module License end def definition_for(name, version) - uri = URI.parse("https://pypi.org/pypi/#{name}/#{version}/json") - definition = process(http.with_retry { |client| client.get(uri, headers: { 'Host' => uri.host }) }) - log_info([name, version, definition["license"]].inspect) - definition + uri = "https://pypi.org/pypi/#{name}/#{version}/json" + process(http.get(uri)).tap do |definition| + log_info([name, version, definition["license"]].inspect) + end rescue *Net::Hippie::CONNECTION_ERRORS {} end class << self - #def definition(name, version) - #@pypi ||= new - #@pypi.definition_for(name, version) - #end - def definition(name, version) - response = request("https://pypi.org/pypi/#{name}/#{version}/json") - response.is_a?(Net::HTTPSuccess) ? JSON.parse(response.body).fetch('info', {}) : {} - rescue *Net::Hippie::CONNECTION_ERRORS - {} - end - - def request(location, limit = 10) - uri = URI(location) - http = Net::HTTP.new(uri.host, uri.port) - http.use_ssl = true - response = http.get(uri.request_uri).response - response.is_a?(Net::HTTPRedirection) && limit.positive? ? request(response['location'], limit - 1) : response + @pypi ||= new + @pypi.definition_for(name, version) end end @@ -53,23 +38,12 @@ module License end def default_http - @default_http ||= Net::Hippie::Client.new(headers: default_headers).tap do |client| + @default_http ||= Net::Hippie::Client.new.tap do |client| client.logger = ::Logger.new('http.log') client.follow_redirects = 3 end end - def user_agent - "https://gitlab.com/gitlab-org/security-products/license-management #{License::Management::VERSION}" - end - - def default_headers - { - 'User-Agent' => user_agent, - 'Accept' => '*/*', - } - end - def ok?(response) response.is_a?(Net::HTTPSuccess) end diff --git a/license-management.gemspec b/license-management.gemspec index 46278f0..492fd2d 100644 --- a/license-management.gemspec +++ b/license-management.gemspec @@ -28,6 +28,6 @@ Gem::Specification.new do |spec| spec.require_paths = ['lib'] spec.add_dependency 'license_finder', '~> 5.11' - spec.add_dependency 'net-hippie', '~> 0.2' + spec.add_dependency 'net-hippie', '~> 0.3' spec.add_development_dependency 'rspec', '~> 3.9' end -- cgit v1.2.3 From ad09f31e9a846687ff692c2809f14cecd9136354 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 16:02:55 -0700 Subject: Centralize access to HTTP to turn on airgap mode easily --- lib/license/management.rb | 9 +++++++++ lib/license/management/pypi.rb | 14 +++----------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/lib/license/management.rb b/lib/license/management.rb index a6986e7..85ef4cc 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -14,6 +14,8 @@ require 'license/management/pypi' # This applies a monkey patch to the JsonReport found in the `license_finder` gem. LicenseFinder::JsonReport.prepend(License::Management::Report) + +# This monkey patch can be removed once https://github.com/pivotal/LicenseFinder/pull/659 is released LicenseFinder::Scanner.const_set( :PACKAGE_MANAGERS, LicenseFinder::Scanner::PACKAGE_MANAGERS + [License::Management::Pipenv] @@ -38,5 +40,12 @@ module License def self.root Pathname.new(File.dirname(__FILE__)).join('../..') end + + def self.http + @http ||= Net::Hippie::Client.new.tap do |client| + client.logger = ::Logger.new('http.log') + client.follow_redirects = 3 + end + end end end diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb index c0254e6..0397532 100644 --- a/lib/license/management/pypi.rb +++ b/lib/license/management/pypi.rb @@ -6,15 +6,14 @@ module License module Management class PyPI include Loggable - attr_reader :http - def initialize(http = default_http) + def initialize(http) @http = http end def definition_for(name, version) uri = "https://pypi.org/pypi/#{name}/#{version}/json" - process(http.get(uri)).tap do |definition| + process(@http.with_retry { |client| client.get(uri) }).tap do |definition| log_info([name, version, definition["license"]].inspect) end rescue *Net::Hippie::CONNECTION_ERRORS @@ -23,7 +22,7 @@ module License class << self def definition(name, version) - @pypi ||= new + @pypi ||= new(License::Management.http) @pypi.definition_for(name, version) end end @@ -37,13 +36,6 @@ module License {} end - def default_http - @default_http ||= Net::Hippie::Client.new.tap do |client| - client.logger = ::Logger.new('http.log') - client.follow_redirects = 3 - end - end - def ok?(response) response.is_a?(Net::HTTPSuccess) end -- cgit v1.2.3 From 1b5d49f45739e47830eaea11986e1f64880095a1 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 16:49:42 -0700 Subject: Move files to python folder --- lib/license/management.rb | 4 +-- lib/license/management/pipenv.rb | 64 --------------------------------- lib/license/management/pypi.rb | 44 ----------------------- lib/license/management/python/pipenv.rb | 64 +++++++++++++++++++++++++++++++++ lib/license/management/python/pypi.rb | 44 +++++++++++++++++++++++ 5 files changed, 110 insertions(+), 110 deletions(-) delete mode 100644 lib/license/management/pipenv.rb delete mode 100644 lib/license/management/pypi.rb create mode 100644 lib/license/management/python/pipenv.rb create mode 100644 lib/license/management/python/pypi.rb diff --git a/lib/license/management.rb b/lib/license/management.rb index 85ef4cc..ebab5e2 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -6,11 +6,11 @@ require 'json' require 'license_finder' require 'license/management/loggable' require 'license/management/verifiable' +require 'license/management/python/pipenv' +require 'license/management/python/pypi' require 'license/management/repository' require 'license/management/report' require 'license/management/version' -require 'license/management/pipenv' -require 'license/management/pypi' # This applies a monkey patch to the JsonReport found in the `license_finder` gem. LicenseFinder::JsonReport.prepend(License::Management::Report) diff --git a/lib/license/management/pipenv.rb b/lib/license/management/pipenv.rb deleted file mode 100644 index 482fd25..0000000 --- a/lib/license/management/pipenv.rb +++ /dev/null @@ -1,64 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - class Pipenv < LicenseFinder::PackageManager - include Loggable - - def initialize(options = {}) - super - @lockfile = Pathname('Pipfile.lock') - end - - def current_packages - @current_packages ||= - begin - packages = {} - each_dependency(groups: allowed_groups) do |name, data, group| - version = canonicalize(data['version']) - package = packages.fetch(key_for(name, version)) do |key| - packages[key] = build_package_for(name, version) - end - package.groups << group - end - packages.values - end - end - - def possible_package_paths - project_path ? [project_path.join(@lockfile)] : [@lockfile] - end - - private - - def each_dependency(groups: []) - dependencies = JSON.parse(IO.read(detected_package_path)) - groups.each do |group| - dependencies[group].each do |name, data| - yield name, data, group - end - end - end - - def canonicalize(version) - version.sub(/^==/, '') - end - - def build_package_for(name, version) - LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version)) - end - - def key_for(name, version) - "#{name}-#{version}" - end - - def allowed_groups - %w[default develop] - ignored_groups - end - - def ignored_groups - @ignored_groups.to_a || [] - end - end - end -end diff --git a/lib/license/management/pypi.rb b/lib/license/management/pypi.rb deleted file mode 100644 index 0397532..0000000 --- a/lib/license/management/pypi.rb +++ /dev/null @@ -1,44 +0,0 @@ -# frozen_string_literal: true - -require 'net/hippie' - -module License - module Management - class PyPI - include Loggable - - def initialize(http) - @http = http - end - - def definition_for(name, version) - uri = "https://pypi.org/pypi/#{name}/#{version}/json" - process(@http.with_retry { |client| client.get(uri) }).tap do |definition| - log_info([name, version, definition["license"]].inspect) - end - rescue *Net::Hippie::CONNECTION_ERRORS - {} - end - - class << self - def definition(name, version) - @pypi ||= new(License::Management.http) - @pypi.definition_for(name, version) - end - end - - private - - def process(response) - return JSON.parse(response.body).fetch('info', {}) if ok?(response) - - log_error([response.class, response.code, response.body].inspect) - {} - end - - def ok?(response) - response.is_a?(Net::HTTPSuccess) - end - end - end -end diff --git a/lib/license/management/python/pipenv.rb b/lib/license/management/python/pipenv.rb new file mode 100644 index 0000000..482fd25 --- /dev/null +++ b/lib/license/management/python/pipenv.rb @@ -0,0 +1,64 @@ +# frozen_string_literal: true + +module License + module Management + class Pipenv < LicenseFinder::PackageManager + include Loggable + + def initialize(options = {}) + super + @lockfile = Pathname('Pipfile.lock') + end + + def current_packages + @current_packages ||= + begin + packages = {} + each_dependency(groups: allowed_groups) do |name, data, group| + version = canonicalize(data['version']) + package = packages.fetch(key_for(name, version)) do |key| + packages[key] = build_package_for(name, version) + end + package.groups << group + end + packages.values + end + end + + def possible_package_paths + project_path ? [project_path.join(@lockfile)] : [@lockfile] + end + + private + + def each_dependency(groups: []) + dependencies = JSON.parse(IO.read(detected_package_path)) + groups.each do |group| + dependencies[group].each do |name, data| + yield name, data, group + end + end + end + + def canonicalize(version) + version.sub(/^==/, '') + end + + def build_package_for(name, version) + LicenseFinder::PipPackage.new(name, version, PyPI.definition(name, version)) + end + + def key_for(name, version) + "#{name}-#{version}" + end + + def allowed_groups + %w[default develop] - ignored_groups + end + + def ignored_groups + @ignored_groups.to_a || [] + end + end + end +end diff --git a/lib/license/management/python/pypi.rb b/lib/license/management/python/pypi.rb new file mode 100644 index 0000000..0397532 --- /dev/null +++ b/lib/license/management/python/pypi.rb @@ -0,0 +1,44 @@ +# frozen_string_literal: true + +require 'net/hippie' + +module License + module Management + class PyPI + include Loggable + + def initialize(http) + @http = http + end + + def definition_for(name, version) + uri = "https://pypi.org/pypi/#{name}/#{version}/json" + process(@http.with_retry { |client| client.get(uri) }).tap do |definition| + log_info([name, version, definition["license"]].inspect) + end + rescue *Net::Hippie::CONNECTION_ERRORS + {} + end + + class << self + def definition(name, version) + @pypi ||= new(License::Management.http) + @pypi.definition_for(name, version) + end + end + + private + + def process(response) + return JSON.parse(response.body).fetch('info', {}) if ok?(response) + + log_error([response.class, response.code, response.body].inspect) + {} + end + + def ok?(response) + response.is_a?(Net::HTTPSuccess) + end + end + end +end -- cgit v1.2.3 From 84f457abde3a17dad26973f1f1c7147eaf58d02b Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 16:50:02 -0700 Subject: Update version constraint --- Gemfile.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gemfile.lock b/Gemfile.lock index b8c2990..abe9075 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -3,7 +3,7 @@ PATH specs: license-management (2.4.0) license_finder (~> 5.11) - net-hippie (~> 0.2) + net-hippie (~> 0.3) GEM remote: https://rubygems.org/ -- cgit v1.2.3 From dbe8cbc4c5fb8c12de236c946dcc82f85df6646b Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 14 Jan 2020 16:50:25 -0700 Subject: Upgrade system packages --- Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 040ab84..f8c0f42 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,11 +14,12 @@ RUN npm install npm-install-peers # install realpath, includes for python3, and pip for python3 # Install additional php packages for better composer package support RUN add-apt-repository ppa:ondrej/php -y && apt-get update -y && \ + apt-get upgrade -y --no-install-recommends && \ apt-get install -y --no-install-recommends \ bsdmainutils \ libjpeg8-dev \ zlib1g-dev \ - libpq-dev libmysqlclient-dev realpath dotnet-sdk-2.2 dotnet-sdk-3.0 \ + libpq-dev libmysqlclient-dev realpath \ php7.1-mbstring php7.1-intl php7.1-xml php7.1-soap -y && \ git clone --depth 1 --branch v0.7.6 https://github.com/asdf-vm/asdf.git $HOME/.asdf && \ echo 'pip' >> $HOME/.default-python-packages && \ -- cgit v1.2.3 From 0045121115a0fdad70ca30f4a1e046e53986e02c Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 13:49:58 -0700 Subject: Fix extends in job description --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 0a29b68..e4dcf53 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -153,7 +153,7 @@ QA:python3-pipenv: QA_REF: pip-file-lock QA:python3-pipenv-v1-1: - extends: QA:python3-pip + extends: QA:python3-pipenv variables: LM_REPORT_VERSION: '1.1' QA_RESULTS: python-pipenv-v1.1 -- cgit v1.2.3 From 3aeed26912b238f6cb9c8b43587b0f3a7bf181f4 Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 14:19:39 -0700 Subject: Add spec to ensure we can parse a v3 Pipfile.lock --- Gemfile.lock | 6 ++++ license-management.gemspec | 1 + spec/fixtures/v2.0_schema.json | 16 +++++++++++ spec/integration/python/pipenv_spec.rb | 51 ++++++++++++++++++++++++++++++++-- spec/spec_helper.rb | 2 ++ spec/support/matchers.rb | 13 +++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 spec/fixtures/v2.0_schema.json create mode 100644 spec/support/matchers.rb diff --git a/Gemfile.lock b/Gemfile.lock index abe9075..60f69d1 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,7 +8,11 @@ PATH GEM remote: https://rubygems.org/ specs: + addressable (2.7.0) + public_suffix (>= 2.0.2, < 5.0) diff-lcs (1.3) + json-schema (2.8.1) + addressable (>= 2.4) license_finder (5.11.1) bundler rubyzip (>= 1, < 3) @@ -18,6 +22,7 @@ GEM xml-simple net-hippie (0.3.0) parslet (1.8.2) + public_suffix (4.0.3) rspec (3.9.0) rspec-core (~> 3.9.0) rspec-expectations (~> 3.9.0) @@ -42,6 +47,7 @@ PLATFORMS ruby DEPENDENCIES + json-schema (~> 2.8) license-management! rspec (~> 3.9) diff --git a/license-management.gemspec b/license-management.gemspec index 492fd2d..60ab5e1 100644 --- a/license-management.gemspec +++ b/license-management.gemspec @@ -29,5 +29,6 @@ Gem::Specification.new do |spec| spec.add_dependency 'license_finder', '~> 5.11' spec.add_dependency 'net-hippie', '~> 0.3' + spec.add_development_dependency 'json-schema', '~> 2.8' spec.add_development_dependency 'rspec', '~> 3.9' end diff --git a/spec/fixtures/v2.0_schema.json b/spec/fixtures/v2.0_schema.json new file mode 100644 index 0000000..bd304ce --- /dev/null +++ b/spec/fixtures/v2.0_schema.json @@ -0,0 +1,16 @@ +{ + "$id": "https://gitlab.com/gitlab-org/security-products/license-management/blob/master/spec/fixtures/v2.0_schema.json", + "type": "object", + "required": [ + "version", + "licenses", + "dependencies" + ], + "properties": { + "version": { "type": "string" }, + "licenses": { "type": "array" }, + "dependencies": { "type": "array" } + }, + "additionalProperties": false +} + diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb index 15e48dc..6039b25 100644 --- a/spec/integration/python/pipenv_spec.rb +++ b/spec/integration/python/pipenv_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' RSpec.describe "pipenv" do - context "when a project depends on a Pipfile.lock" do + context "when a project depends on a version 6 Pipfile.lock" do let(:pipfile_lock_content) do JSON.pretty_generate({ "_meta": { @@ -23,8 +23,55 @@ RSpec.describe "pipenv" do report = runner.scan expect(report).not_to be_empty - expect(report[:version]).to start_with('2') + expect(report[:version]).not_to be_empty + expect(report[:licenses]).not_to be_empty expect(report[:dependencies].map { |x| x[:name] }).to include("six") end end + + context "when a project depends on a version 3.2.1 Pipfile.lock" do + let(:pipfile_lock_content) do + JSON.pretty_generate({ + "default": { + "crayons": { "version": "==0.1.2", "hash": "" }, + "requirements-parser": { "version": "==0.1.0", "hash": "" }, + "pexpect": { "version": "==4.2.1", "hash": "" }, + "delegator.py": { "version": "==0.0.8", "hash": "" }, + "backports.shutil_get_terminal_size": { "version": "==1.0.0", "hash": "" }, + "ptyprocess": { "version": "==0.5.1", "hash": "" }, + "parse": { "version": "==1.6.6", "hash": "" }, + "toml": { "version": "==0.9.2", "hash": "" }, + "colorama": { "version": "==0.3.7", "hash": "" }, + "requests": { "version": "==2.13.0", "hash": "" }, + "click": { "version": "==6.7", "hash": "" } + }, + "develop": { + "packaging": { "version": "==16.8", "hash": "" }, + "pytest": { "version": "==3.0.6", "hash": "" }, + "setuptools": { "version": "==34.0.2", "hash": "" }, + "pyparsing": { "version": "==2.1.10", "hash": "" }, + "py": { "version": "==1.4.32", "hash": "" }, + "six": { "version": "==1.10.0", "hash": "" }, + "appdirs": { "version": "==1.4.0", "hash": "" } + }, + "_meta": { + "sources": [ { "url": "https://pypi.python.org/simple", "verify_ssl": true } ], + "requires": {}, + "Pipfile-sha256": "24f12b631b7c40b8c5eff934a1aef263ed04f5eaffb4acf4706442f3d23cba36" + } + }) + end + + it 'produces a valid report' do + runner.add_file('Pipfile.lock', pipfile_lock_content) + + report = runner.scan + + expect(report).to match_schema(version: '2.0') + expect(report).not_to be_empty + expect(report[:version]).not_to be_empty + expect(report[:licenses]).not_to be_empty + expect(report[:dependencies].count).to eql(18) + end + end end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 115822b..1889335 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,7 +1,9 @@ require 'license/management' require 'json' require 'securerandom' +require 'json-schema' require 'support/integration_test_helper' +require 'support/matchers' RSpec.configure do |config| config.include IntegrationTestHelper, type: :integration diff --git a/spec/support/matchers.rb b/spec/support/matchers.rb new file mode 100644 index 0000000..bb54d19 --- /dev/null +++ b/spec/support/matchers.rb @@ -0,0 +1,13 @@ +RSpec::Matchers.define :match_schema do |version: nil, **options| + match do |actual| + path = License::Management.root.join("spec/fixtures/v#{version}_schema.json") + schema = JSON.parse(IO.read(path)) + @errors = JSON::Validator.fully_validate(schema, actual, options) + @errors.empty? + end + + failure_message do |response| + "didn't match the schema for version #{version}" \ + " The validation errors were:\n#{@errors.join("\n")}" + end +end -- cgit v1.2.3 From ac9634d8f8c6bbaa8348929f42d53d2b375f6dfb Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 14:41:14 -0700 Subject: Add a json schema for dependency --- spec/fixtures/schema/v2.0.json | 16 ++++++++++++++++ spec/fixtures/schema/v2.0_dependency.json | 27 +++++++++++++++++++++++++++ spec/fixtures/v2.0_schema.json | 16 ---------------- spec/integration/python/pipenv_spec.rb | 1 + spec/integration/ruby/bundler_spec.rb | 2 ++ spec/support/matchers.rb | 9 +++++---- 6 files changed, 51 insertions(+), 20 deletions(-) create mode 100644 spec/fixtures/schema/v2.0.json create mode 100644 spec/fixtures/schema/v2.0_dependency.json delete mode 100644 spec/fixtures/v2.0_schema.json diff --git a/spec/fixtures/schema/v2.0.json b/spec/fixtures/schema/v2.0.json new file mode 100644 index 0000000..3377a9d --- /dev/null +++ b/spec/fixtures/schema/v2.0.json @@ -0,0 +1,16 @@ +{ + "$id": "https://gitlab.com/gitlab-org/security-products/license-management/blob/master/spec/fixtures/schema/v2.0.json", + "type": "object", + "required": [ + "version", + "licenses", + "dependencies" + ], + "properties": { + "version": { "type": "string" }, + "licenses": { "type": "array" }, + "dependencies": { "$ref": "v2.0_dependency.json" } + }, + "additionalProperties": false +} + diff --git a/spec/fixtures/schema/v2.0_dependency.json b/spec/fixtures/schema/v2.0_dependency.json new file mode 100644 index 0000000..fcd96d7 --- /dev/null +++ b/spec/fixtures/schema/v2.0_dependency.json @@ -0,0 +1,27 @@ +{ + "type": "array", + "required": [ + "name", + "url", + "description", + "paths", + "licenses" + ], + "properties": { + "name": { "type": "string" }, + "url": { "type": "uri" }, + "description": { "type": "string" }, + "paths": { + "type": "array", + "items": { + "type": "string" + } + }, + "licenses": { + "type": "array", + "items": { + "type": "string" + } + } + } +} diff --git a/spec/fixtures/v2.0_schema.json b/spec/fixtures/v2.0_schema.json deleted file mode 100644 index bd304ce..0000000 --- a/spec/fixtures/v2.0_schema.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "$id": "https://gitlab.com/gitlab-org/security-products/license-management/blob/master/spec/fixtures/v2.0_schema.json", - "type": "object", - "required": [ - "version", - "licenses", - "dependencies" - ], - "properties": { - "version": { "type": "string" }, - "licenses": { "type": "array" }, - "dependencies": { "type": "array" } - }, - "additionalProperties": false -} - diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb index 6039b25..2756f6b 100644 --- a/spec/integration/python/pipenv_spec.rb +++ b/spec/integration/python/pipenv_spec.rb @@ -23,6 +23,7 @@ RSpec.describe "pipenv" do report = runner.scan expect(report).not_to be_empty + expect(report).to match_schema(version: '2.0') expect(report[:version]).not_to be_empty expect(report[:licenses]).not_to be_empty expect(report[:dependencies].map { |x| x[:name] }).to include("six") diff --git a/spec/integration/ruby/bundler_spec.rb b/spec/integration/ruby/bundler_spec.rb index 179da2a..2cb8f7d 100644 --- a/spec/integration/ruby/bundler_spec.rb +++ b/spec/integration/ruby/bundler_spec.rb @@ -14,6 +14,7 @@ gem 'saml-kit' report = runner.scan expect(report).not_to be_empty + expect(report).to match_schema(version: '2.0') expect(report[:licenses]).not_to be_empty expect(report[:dependencies].map { |x| x[:name] }).to include("saml-kit") end @@ -81,6 +82,7 @@ BUNDLED WITH report = runner.scan expect(report).not_to be_empty + expect(report).to match_schema(version: '2.0') expect(report[:licenses]).not_to be_empty expect(report[:dependencies].map { |x| x[:name] }).to include("saml-kit") end diff --git a/spec/support/matchers.rb b/spec/support/matchers.rb index bb54d19..1d1c263 100644 --- a/spec/support/matchers.rb +++ b/spec/support/matchers.rb @@ -1,8 +1,9 @@ -RSpec::Matchers.define :match_schema do |version: nil, **options| +RSpec::Matchers.define :match_schema do |version: '2.0'| match do |actual| - path = License::Management.root.join("spec/fixtures/v#{version}_schema.json") - schema = JSON.parse(IO.read(path)) - @errors = JSON::Validator.fully_validate(schema, actual, options) + schema = License::Management.root + .join("spec/fixtures/schema/v#{version}.json") + .to_s + @errors = JSON::Validator.fully_validate(schema, actual) @errors.empty? end -- cgit v1.2.3 From 3d4fbad1e39f7cbb37643864b8229aa4116ce07c Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 14:51:53 -0700 Subject: Combine schema into a single file --- spec/fixtures/schema/v2.0.json | 48 +++++++++++++++++++++++++++++-- spec/fixtures/schema/v2.0_dependency.json | 27 ----------------- 2 files changed, 46 insertions(+), 29 deletions(-) delete mode 100644 spec/fixtures/schema/v2.0_dependency.json diff --git a/spec/fixtures/schema/v2.0.json b/spec/fixtures/schema/v2.0.json index 3377a9d..712143b 100644 --- a/spec/fixtures/schema/v2.0.json +++ b/spec/fixtures/schema/v2.0.json @@ -8,8 +8,52 @@ ], "properties": { "version": { "type": "string" }, - "licenses": { "type": "array" }, - "dependencies": { "$ref": "v2.0_dependency.json" } + "licenses": { + "type": "array", + "items": { + "type": "object", + "required": [ + "id", + "name", + "url" + ], + "properties": { + "id": { "type": "string" }, + "name": { "type": "string" }, + "url": { "type": "uri" } + } + } + }, + "dependencies": { + "type": "array", + "items": { + "type": "object", + "required": [ + "name", + "url", + "description", + "paths", + "licenses" + ], + "properties": { + "name": { "type": "string" }, + "url": { "type": "uri" }, + "description": { "type": "string" }, + "paths": { + "type": "array", + "items": { + "type": "string" + } + }, + "licenses": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + } }, "additionalProperties": false } diff --git a/spec/fixtures/schema/v2.0_dependency.json b/spec/fixtures/schema/v2.0_dependency.json deleted file mode 100644 index fcd96d7..0000000 --- a/spec/fixtures/schema/v2.0_dependency.json +++ /dev/null @@ -1,27 +0,0 @@ -{ - "type": "array", - "required": [ - "name", - "url", - "description", - "paths", - "licenses" - ], - "properties": { - "name": { "type": "string" }, - "url": { "type": "uri" }, - "description": { "type": "string" }, - "paths": { - "type": "array", - "items": { - "type": "string" - } - }, - "licenses": { - "type": "array", - "items": { - "type": "string" - } - } - } -} -- cgit v1.2.3 From 8caa7cac09bfacacc3e0eb4ea8c7adc63f48ff08 Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 14:56:59 -0700 Subject: Print the image size --- .gitlab-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e4dcf53..aa8592b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,6 +56,7 @@ size: script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $TMP_IMAGE + - docker image ls unit: stage: test -- cgit v1.2.3 From 1046fecb2ea48d6fac210502dbe941725a1f5a71 Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 15:01:41 -0700 Subject: Inspect the image details --- .gitlab-ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index aa8592b..626153b 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -56,7 +56,8 @@ size: script: - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $TMP_IMAGE - - docker image ls + - docker image ls $TMP_IMAGE + - docker image inspect $TMP_IMAGE unit: stage: test -- cgit v1.2.3 From 3e9bbcd897812e64d3a356dcd5806baeec198d65 Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 15:11:58 -0700 Subject: Validate version # --- spec/fixtures/schema/v2.0.json | 30 ++++++++++++++++++++++-------- spec/integration/dotnet/examples_spec.rb | 1 + spec/integration/python/pip_spec.rb | 2 ++ 3 files changed, 25 insertions(+), 8 deletions(-) diff --git a/spec/fixtures/schema/v2.0.json b/spec/fixtures/schema/v2.0.json index 712143b..6ade7ad 100644 --- a/spec/fixtures/schema/v2.0.json +++ b/spec/fixtures/schema/v2.0.json @@ -7,7 +7,10 @@ "dependencies" ], "properties": { - "version": { "type": "string" }, + "version": { + "type": "string", + "pattern": "^[0-9]+\\.[0-9]+$" + }, "licenses": { "type": "array", "items": { @@ -18,9 +21,15 @@ "url" ], "properties": { - "id": { "type": "string" }, - "name": { "type": "string" }, - "url": { "type": "uri" } + "id": { + "type": "string" + }, + "name": { + "type": "string" + }, + "url": { + "type": "uri" + } } } }, @@ -36,9 +45,15 @@ "licenses" ], "properties": { - "name": { "type": "string" }, - "url": { "type": "uri" }, - "description": { "type": "string" }, + "name": { + "type": "string" + }, + "url": { + "type": "uri" + }, + "description": { + "type": "string" + }, "paths": { "type": "array", "items": { @@ -57,4 +72,3 @@ }, "additionalProperties": false } - diff --git a/spec/integration/dotnet/examples_spec.rb b/spec/integration/dotnet/examples_spec.rb index 43caea0..5b2a913 100644 --- a/spec/integration/dotnet/examples_spec.rb +++ b/spec/integration/dotnet/examples_spec.rb @@ -6,6 +6,7 @@ RSpec.describe ".NET Core" do report = runner.scan(env: { 'LICENSE_FINDER_CLI_OPTS' => '--recursive' }) expect(report).not_to be_empty + expect(report).to match_schema(version: '2.0') expect(report[:licenses].count).not_to be_zero expect(report[:dependencies].count).not_to be_zero end diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb index 5ff5f60..b092189 100644 --- a/spec/integration/python/pip_spec.rb +++ b/spec/integration/python/pip_spec.rb @@ -10,6 +10,7 @@ RSpec.describe "pip" do report = runner.scan expect(report).not_to be_empty + expect(report).to match_schema(version: '2.0') expect(report[:version]).to start_with('2') expect(report[:dependencies].map { |x| x[:name] }).to include("sentry-sdk") expect(report[:dependencies].find { |x| x[:name] == 'sentry-sdk' }[:licenses]).to match_array(["BSD-4-Clause"]) @@ -39,6 +40,7 @@ RSpec.describe "pip" do report = runner.scan expect(report).not_to be_empty + expect(report).to match_schema(version: '2.0') expect(report[:version]).to start_with('2') expect(report[:licenses]).not_to be_empty expect(report[:dependencies]).not_to be_empty -- cgit v1.2.3 From 06d2844938465c84f908139db7e7a3abc4e3cd6d Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 15:30:44 -0700 Subject: Parse version 5 of the pipfile.lock --- spec/integration/python/pipenv_spec.rb | 49 ++++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb index 2756f6b..4d6559c 100644 --- a/spec/integration/python/pipenv_spec.rb +++ b/spec/integration/python/pipenv_spec.rb @@ -75,4 +75,53 @@ RSpec.describe "pipenv" do expect(report[:dependencies].count).to eql(18) end end + + context "when a project depends on a version 5 Pipfile.lock" do + let(:pipfile_lock_content) do + JSON.pretty_generate({ + "_meta": { + "hash": { "sha256": "" }, + "host-environment-markers": { + "implementation_name": "cpython", + "implementation_version": "3.6.1", + "os_name": "posix", + "platform_machine": "x86_64", + "platform_python_implementation": "CPython", + "platform_release": "16.7.0", + "platform_system": "Darwin", + "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64", + "python_full_version": "3.6.1", + "python_version": "3.6", + "sys_platform": "darwin" + }, + "pipfile-spec": 5, + "requires": {}, + "sources": [{ "name": "pypi", "url": "https://pypi.python.org/simple", "verify_ssl": true }] + }, + "default": { + "certifi": { "hashes": ["", ""], "version": "==2017.7.27.1" }, + "chardet": { "hashes": ["", ""], "version": "==3.0.4" }, + "idna": { "hashes": ["", ""], "version": "==2.6" }, + "requests": { "hashes": ["", ""], "version": "==2.18.4" }, + "urllib3": { "hashes": ["", ""], "version": "==1.22" } + }, + "develop": { + "py": { "hashes": ["", ""], "version": "==1.4.34" }, + "pytest": { "hashes": ["", ""], "version": "==3.2.2" } + } + }) + end + + it 'produces a valid report' do + runner.add_file('Pipfile.lock', pipfile_lock_content) + + report = runner.scan + + expect(report).to match_schema(version: '2.0') + expect(report).not_to be_empty + expect(report[:version]).not_to be_empty + expect(report[:licenses]).not_to be_empty + expect(report[:dependencies].count).to eql(7) + end + end end -- cgit v1.2.3 From 34f162a4903d852d47bd5440839f7519eb6fa8f0 Mon Sep 17 00:00:00 2001 From: mo khan Date: Wed, 15 Jan 2020 16:25:16 -0700 Subject: Match dependency names --- spec/integration/python/pipenv_spec.rb | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb index 4d6559c..1ea60fe 100644 --- a/spec/integration/python/pipenv_spec.rb +++ b/spec/integration/python/pipenv_spec.rb @@ -26,7 +26,7 @@ RSpec.describe "pipenv" do expect(report).to match_schema(version: '2.0') expect(report[:version]).not_to be_empty expect(report[:licenses]).not_to be_empty - expect(report[:dependencies].map { |x| x[:name] }).to include("six") + expect(report[:dependencies].map { |x| x[:name] }).to contain_exactly("six") end end @@ -72,7 +72,26 @@ RSpec.describe "pipenv" do expect(report).not_to be_empty expect(report[:version]).not_to be_empty expect(report[:licenses]).not_to be_empty - expect(report[:dependencies].count).to eql(18) + expect(report[:dependencies].map { |x| x[:name] }).to match_array([ + "appdirs", + "backports.shutil_get_terminal_size", + "click", + "colorama", + "crayons", + "delegator.py", + "packaging", + "parse", + "pexpect", + "ptyprocess", + "py", + "pyparsing", + "pytest", + "requests", + "requirements-parser", + "setuptools", + "six", + "toml", + ]) end end @@ -121,7 +140,9 @@ RSpec.describe "pipenv" do expect(report).not_to be_empty expect(report[:version]).not_to be_empty expect(report[:licenses]).not_to be_empty - expect(report[:dependencies].count).to eql(7) + expect(report[:dependencies].map { |x| x[:name] }).to match_array([ + 'certifi', 'chardet', 'idna', 'requests', 'urllib3', 'py', 'pytest' + ]) end end end -- cgit v1.2.3