diff options
| -rw-r--r-- | CHANGELOG.md | 5 | ||||
| -rw-r--r-- | Gemfile.lock | 2 | ||||
| -rw-r--r-- | config/.default-python-packages | 1 | ||||
| -rw-r--r-- | lib/license/finder/ext.rb | 1 | ||||
| -rw-r--r-- | lib/license/finder/ext/pip.rb | 46 | ||||
| -rw-r--r-- | lib/license/finder/ext/pipenv.rb | 63 | ||||
| -rw-r--r-- | lib/license/management.rb | 1 | ||||
| -rw-r--r-- | lib/license/management/python.rb | 52 | ||||
| -rw-r--r-- | lib/license/management/version.rb | 2 | ||||
| -rw-r--r-- | normalized-licenses.yml | 1 | ||||
| -rwxr-xr-x | run.sh | 5 | ||||
| -rw-r--r-- | spec/fixtures/expected/python/pipenv/v1.0.json | 2 | ||||
| -rw-r--r-- | spec/fixtures/expected/python/pipenv/v1.1.json | 2 | ||||
| -rw-r--r-- | spec/fixtures/expected/python/pipenv/v2.0.json | 2 | ||||
| -rw-r--r-- | spec/fixtures/python/simple-Pipfile | 10 | ||||
| -rw-r--r-- | spec/fixtures/python/simple-Pipfile.lock | 69 | ||||
| -rw-r--r-- | spec/integration/python/pipenv_spec.rb | 69 | ||||
| -rw-r--r-- | spec/support/integration_test_helper.rb | 2 |
18 files changed, 283 insertions, 52 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 98beed6..ff5f1ed 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # GitLab License management changelog +## v3.4.0 + +- Scan pipenv projects with [pip-licenses](https://pypi.org/project/pip-licenses/). (!130) +- Read pipenv spec data from the sources listed in `Pipfile.lock`. (!130) + ## v3.3.1 - Fix bug with forwarding `LICENSE_FINDER_CLI_OPTS` (!131) diff --git a/Gemfile.lock b/Gemfile.lock index a8a1160..f4f4afa 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - license-management (3.3.1) + license-management (3.4.0) license_finder (~> 6.0.0) spandx (~> 0.1) diff --git a/config/.default-python-packages b/config/.default-python-packages index 86cadc4..ddf6574 100644 --- a/config/.default-python-packages +++ b/config/.default-python-packages @@ -1,3 +1,4 @@ conan pip +pipenv virtualenv diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb index 8731e4f..fffa1c7 100644 --- a/lib/license/finder/ext.rb +++ b/lib/license/finder/ext.rb @@ -4,6 +4,7 @@ require 'license/finder/ext/license' require 'license/finder/ext/maven' require 'license/finder/ext/nuget' require 'license/finder/ext/pip' +require 'license/finder/ext/pipenv' require 'license/finder/ext/shared_helpers' # Apply patch to the JsonReport found in the `license_finder` gem. diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb index e83f64c..b57d7c8 100644 --- a/lib/license/finder/ext/pip.rb +++ b/lib/license/finder/ext/pip.rb @@ -5,18 +5,8 @@ module LicenseFinder def current_packages return legacy_results unless virtual_env? - _stdout, _stderr, status = pip_licenses - return legacy_results unless status.success? - - JSON.parse(IO.read('pip-licenses.json')).map do |dependency| - Package.new( - dependency['Name'], - dependency['Version'], - description: dependency['Description'], - homepage: dependency['URL'], - spec_licenses: [dependency['License']] - ) - end + dependencies = python.pip_licenses + dependencies.any? ? dependencies : legacy_results end def possible_package_paths @@ -38,39 +28,23 @@ module LicenseFinder private + def python + @python ||= ::License::Management::Python.new + end + def install_packages within_project_dir do - shell.execute(['virtualenv -p', python_executable, '--activators=bash --seeder=app-data venv']) - shell.sh([". venv/bin/activate", "&&", :pip, :install, '-i', pip_index_url, '-r', @requirements_path]) + shell.execute(['virtualenv -p', python_executable, '--activators=bash --seeder=app-data .venv']) + shell.sh([". .venv/bin/activate", "&&", :pip, :install, '-i', python.pip_index_url, '-r', @requirements_path]) end end - def pip_licenses - shell.sh([ - ". venv/bin/activate &&", - :pip, :install, - '--no-index', - '--find-links $HOME/.config/virtualenv/app-data', 'pip-licenses', '&&', - 'pip-licenses', - '--ignore-packages prettytable', - '--with-description', - '--with-urls', - '--from=meta', - '--format=json', - '--output-file pip-licenses.json' - ], env: { 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' }) - end - def python_executable '"$(asdf where python)/bin/python"' end - def pip_index_url - ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/') - end - def virtual_env? - within_project_dir { File.exist?('venv/bin/activate') } + within_project_dir { File.exist?('.venv/bin/activate') } end def within_project_dir @@ -85,7 +59,7 @@ module LicenseFinder @pypi ||= Spandx::Python::PyPI.new(sources: [ Spandx::Python::Source.new({ 'name' => 'pypi', - 'url' => pip_index_url, + 'url' => python.pip_index_url, 'verify_ssl' => true }) ]) diff --git a/lib/license/finder/ext/pipenv.rb b/lib/license/finder/ext/pipenv.rb new file mode 100644 index 0000000..ebcc524 --- /dev/null +++ b/lib/license/finder/ext/pipenv.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +module LicenseFinder + class Pipenv + def prepare + return unless pipfile? + + shell.execute([ + :pipenv, + :install, + '--python', + python.major_version, + '--ignore-pipfile', + '--index', + python.pip_index_url + ]) + end + + def current_packages + return legacy_results unless pipfile? + + python.pip_licenses + end + + private + + def shell + @shell ||= ::License::Management::Shell.new + end + + def python + @python ||= ::License::Management::Python.new + end + + def pipfile? + detected_package_path.dirname.join('Pipfile').exist? + end + + def legacy_results + packages = {} + each_dependency(groups: allowed_groups) do |name, data, group| + version = canonicalize(data['version']) + package = packages.fetch(key_for(name, version)) do |key| + packages[key] = build_package_for(name, version) + end + package.groups << group + end + packages.values + end + + def build_package_for(name, version) + PipPackage.new(name, version, pypi.definition_for(name, version)) + end + + def pypi + @pypi ||= ::Spandx::Python::PyPI.new(sources: ::Spandx::Python::Source.sources_from(lockfile_hash)) + end + + def lockfile_hash + @lockfile_hash ||= JSON.parse(IO.read(detected_package_path)) + end + end +end diff --git a/lib/license/management.rb b/lib/license/management.rb index e7a5b23..930fa08 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -9,6 +9,7 @@ require 'yaml' require 'license_finder' require 'license/management/loggable' require 'license/management/verifiable' +require 'license/management/python' require 'license/management/repository' require 'license/management/report' require 'license/management/shell' diff --git a/lib/license/management/python.rb b/lib/license/management/python.rb new file mode 100644 index 0000000..37771ba --- /dev/null +++ b/lib/license/management/python.rb @@ -0,0 +1,52 @@ +# frozen_string_literal: true + +module License + module Management + class Python + attr_reader :shell + + def initialize(shell: Shell.new) + @shell = shell + end + + def major_version + version.split('.')[0] + end + + def version + ENV.fetch('LM_PYTHON_VERSION', '3') + end + + def pip_index_url + ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/') + end + + def pip_licenses(venv: '.venv') + _stdout, _stderr, status = shell.sh([ + ". #{venv}/bin/activate &&", + :pip, :install, + '--no-index', + '--find-links $HOME/.config/virtualenv/app-data', 'pip-licenses', '&&', + 'pip-licenses', + '--ignore-packages prettytable', + '--with-description', + '--with-urls', + '--from=meta', + '--format=json', + '--output-file pip-licenses.json' + ], env: { 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' }) + return [] unless status.success? + + JSON.parse(IO.read('pip-licenses.json')).map do |dependency| + ::LicenseFinder::Package.new( + dependency['Name'], + dependency['Version'], + description: dependency['Description'], + homepage: dependency['URL'], + spec_licenses: [dependency['License']] + ) + end + end + end + end +end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index e535634..cee6d57 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.3.1' + VERSION = '3.4.0' end end diff --git a/normalized-licenses.yml b/normalized-licenses.yml index b78f389..a77c27f 100644 --- a/normalized-licenses.yml +++ b/normalized-licenses.yml @@ -29,6 +29,7 @@ ids: LGPL2_1: LGPL-2.1 LGPL: LGPL-3.0-only LGPL, version 2.1: LGPL-2.1 + "License :: OSI Approved :: MIT License": MIT MIT: MIT MIT/X11: MIT Mozilla Public License 2.0: MPL-2.0 @@ -11,9 +11,11 @@ export CI_DEBUG_TRACE=${CI_DEBUG_TRACE:='false'} export DOTNET_CLI_TELEMETRY_OPTOUT=1 export HISTFILESIZE=0 export HISTSIZE=0 +export LANG=C.UTF-8 export LICENSE_FINDER_CLI_OPTS=${LICENSE_FINDER_CLI_OPTS:=--no-debug} export LM_REPORT_FILE=${LM_REPORT_FILE:-'gl-license-management-report.json'} export MAVEN_CLI_OPTS="${MAVEN_CLI_OPTS:--DskipTests}" +export PIPENV_VENV_IN_PROJECT=1 export PREPARE="${PREPARE:---prepare-no-fail}" export RECURSIVE='--no-recursive' export RUBY_GC_HEAP_INIT_SLOTS=800000 @@ -45,6 +47,7 @@ function debug_env() { function scan_project() { gem install -f --silent "$LM_HOME/pkg/*.gem" license_management ignored_groups add development + license_management ignored_groups add develop license_management ignored_groups add test echo license_management report "$@" # shellcheck disable=SC2068 @@ -81,7 +84,7 @@ function prepare_dotnet() { function prepare_project() { if [[ -z ${SETUP_CMD:-} ]]; then - asdf install + asdf install 1> /dev/null prepare_javascript || true prepare_golang || true diff --git a/spec/fixtures/expected/python/pipenv/v1.0.json b/spec/fixtures/expected/python/pipenv/v1.0.json index 6c0ae63..89bce2a 100644 --- a/spec/fixtures/expected/python/pipenv/v1.0.json +++ b/spec/fixtures/expected/python/pipenv/v1.0.json @@ -24,7 +24,7 @@ "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29" }, "dependency": { - "name": "django", + "name": "Django", "url": "https://www.djangoproject.com/", "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.", "pathes": [ diff --git a/spec/fixtures/expected/python/pipenv/v1.1.json b/spec/fixtures/expected/python/pipenv/v1.1.json index 0528f88..92a5153 100644 --- a/spec/fixtures/expected/python/pipenv/v1.1.json +++ b/spec/fixtures/expected/python/pipenv/v1.1.json @@ -31,7 +31,7 @@ "url": "http://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29" }, "dependency": { - "name": "django", + "name": "Django", "url": "https://www.djangoproject.com/", "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.", "pathes": [ diff --git a/spec/fixtures/expected/python/pipenv/v2.0.json b/spec/fixtures/expected/python/pipenv/v2.0.json index bdbeb14..ba4d529 100644 --- a/spec/fixtures/expected/python/pipenv/v2.0.json +++ b/spec/fixtures/expected/python/pipenv/v2.0.json @@ -28,7 +28,7 @@ ], "dependencies": [ { - "name": "django", + "name": "Django", "url": "https://www.djangoproject.com/", "description": "A high-level Python Web framework that encourages rapid development and clean, pragmatic design.", "paths": [ diff --git a/spec/fixtures/python/simple-Pipfile b/spec/fixtures/python/simple-Pipfile new file mode 100644 index 0000000..2a4ffeb --- /dev/null +++ b/spec/fixtures/python/simple-Pipfile @@ -0,0 +1,10 @@ +[[source]] +url = "https://pypi.python.org/simple" +verify_ssl = true +name = "pypi" + +[packages] +requests = "*" + +[dev-packages] +pytest = "*" diff --git a/spec/fixtures/python/simple-Pipfile.lock b/spec/fixtures/python/simple-Pipfile.lock new file mode 100644 index 0000000..655ee42 --- /dev/null +++ b/spec/fixtures/python/simple-Pipfile.lock @@ -0,0 +1,69 @@ +{ + "_meta": { + "hash": { + "sha256": "8d14434df45e0ef884d6c3f6e8048ba72335637a8631cc44792f52fd20b6f97a" + }, + "pipfile-spec": 5, + "requires": {}, + "sources": [ + { + "name": "pypi", + "url": "https://pypi.python.org/simple", + "verify_ssl": true + } + ] + }, + "default": { + "certifi": { + "hashes": [ + "sha256:54a07c09c586b0e4c619f02a5e94e36619da8e2b053e20f594348c0611803704", + "sha256:40523d2efb60523e113b44602298f0960e900388cf3bb6043f645cf57ea9e3f5" + ], + "version": "==2017.7.27.1" + }, + "chardet": { + "hashes": [ + "sha256:fc323ffcaeaed0e0a02bf4d117757b98aed530d9ed4531e3e15460124c106691", + "sha256:84ab92ed1c4d4f16916e05906b6b75a6c0fb5db821cc65e70cbd64a3e2a5eaae" + ], + "version": "==3.0.4" + }, + "idna": { + "hashes": [ + "sha256:8c7309c718f94b3a625cb648ace320157ad16ff131ae0af362c9f21b80ef6ec4", + "sha256:2c6a5de3089009e3da7c5dde64a141dbc8551d5b7f6cf4ed7c2568d0cc520a8f" + ], + "version": "==2.6" + }, + "requests": { + "hashes": [ + "sha256:6a1b267aa90cac58ac3a765d067950e7dbbf75b1da07e895d1f594193a40a38b", + "sha256:9c443e7324ba5b85070c4a818ade28bfabedf16ea10206da1132edaa6dda237e" + ], + "version": "==2.18.4" + }, + "urllib3": { + "hashes": [ + "sha256:06330f386d6e4b195fbfc736b297f58c5a892e4440e54d294d7004e3a9bbea1b", + "sha256:cc44da8e1145637334317feebd728bd869a35285b93cbb4cca2577da7e62db4f" + ], + "version": "==1.22" + } + }, + "develop": { + "py": { + "hashes": [ + "sha256:2ccb79b01769d99115aa600d7eed99f524bf752bba8f041dc1c184853514655a", + "sha256:0f2d585d22050e90c7d293b6451c83db097df77871974d90efd5a30dc12fcde3" + ], + "version": "==1.4.34" + }, + "pytest": { + "hashes": [ + "sha256:b84f554f8ddc23add65c411bf112b2d88e2489fd45f753b1cae5936358bdf314", + "sha256:f46e49e0340a532764991c498244a60e3a37d7424a532b3ff1a6a7653f1a403a" + ], + "version": "==3.2.2" + } + } +} diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb index f0aa0db..983ea8b 100644 --- a/spec/integration/python/pipenv_spec.rb +++ b/spec/integration/python/pipenv_spec.rb @@ -75,23 +75,16 @@ RSpec.describe "pipenv" do expect(report[:version]).not_to be_empty expect(report[:licenses]).not_to be_empty expect(report[:dependencies].map { |x| x[:name] }).to match_array([ - "appdirs", "backports.shutil_get_terminal_size", "click", "colorama", "crayons", "delegator.py", - "packaging", "parse", "pexpect", "ptyprocess", - "py", - "pyparsing", - "pytest", "requests", "requirements-parser", - "setuptools", - "six", "toml" ]) end @@ -146,11 +139,69 @@ RSpec.describe "pipenv" do certifi chardet idna - py - pytest requests urllib3 ]) end end + + context "when fetching metadata from a custom source" do + let(:pipfile_lock_content) do + JSON.pretty_generate({ + "_meta": { + "hash": { "sha256": "" }, + "pipfile-spec": 6, + "requires": { "python_version": "3.8" }, + "sources": [{ "name": "pypi", "url": "https://test.pypi.org/simple", "verify_ssl": true }] + }, + "default": { + "six": { "hashes": [], "index": "pypi", "version": "==1.13.0" } + }, + "develop": {} + }) + end + + before do + runner.add_file('Pipfile.lock', pipfile_lock_content) + end + + it 'produces a valid report' do + report = runner.scan + + expect(report).to match_schema(version: '2.0') + expect(report[:licenses]).not_to be_empty + expect(report[:dependencies].count).to be(1) + expect(find_in(report, 'six')).not_to be_nil + end + end + + context "when scanning a simple Pipfile project" do + let(:lockfile_content) { fixture_file_content('python/simple-Pipfile.lock') } + let(:lockfile_hash) { JSON.parse(lockfile_content) } + + before do + runner.add_file('Pipfile', fixture_file_content('python/simple-Pipfile')) + runner.add_file('Pipfile.lock', lockfile_content) + end + + [2, 3].each do |version| + context "when scanning a Python #{version} project" do + let(:report) { runner.scan(env: { 'LM_PYTHON_VERSION' => version.to_s }) } + + specify { expect(report).to match_schema(version: '2.0') } + + it 'includes dependencies in the default group' do + lockfile_hash['default'].keys.each do |key| + expect(find_in(report, key)).not_to be_nil + end + end + + it 'excludes dependencies in the development group' do + lockfile_hash['develop'].keys.each do |key| + expect(find_in(report, key)).to be_nil + end + end + end + end + end end diff --git a/spec/support/integration_test_helper.rb b/spec/support/integration_test_helper.rb index 25e670f..de04db3 100644 --- a/spec/support/integration_test_helper.rb +++ b/spec/support/integration_test_helper.rb @@ -34,7 +34,7 @@ module IntegrationTestHelper end def execute(env = {}, *args) - Bundler.with_clean_env do + Bundler.with_unbundled_env do system(env, *args, exception: true) end end |