Install pip packages from a custom index

* Read PIP_INDEX_URL to identify where to download packages from

22 files changed, 362 insertions, 7 deletions

diff --git a/Gemfile.lock b/Gemfile.lock
index 8776c4d..068d07f 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -3,6 +3,7 @@ PATH
   specs:
     license-management (3.1.4)
       license_finder (~> 6.0.0)
+      spandx (~> 0.1)
 
 GEM
   remote: https://rubygems.org/
@@ -27,6 +28,10 @@ GEM
       toml (= 0.2.0)
       with_env (= 1.1.0)
       xml-simple
+    mini_portile2 (2.4.0)
+    net-hippie (0.3.2)
+    nokogiri (1.10.9)
+      mini_portile2 (~> 2.4.0)
     parallel (1.19.1)
     parser (2.7.0.4)
       ast (~> 2.4.0)
@@ -67,6 +72,12 @@ GEM
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
     rubyzip (2.3.0)
+    spandx (0.11.0)
+      addressable (~> 2.7)
+      bundler (>= 1.16, < 3.0.0)
+      net-hippie (~> 0.3)
+      nokogiri (~> 1.10)
+      thor
     thor (1.0.1)
     toml (0.2.0)
       parslet (~> 1.8.0)
diff --git a/config/.default-gems b/config/.default-gems
index c41100b..1c3a508 100644
--- a/config/.default-gems
+++ b/config/.default-gems
@@ -1,3 +1,4 @@
 bundler ~>1.7
 bundler ~>2.0
 license_finder ~>6.0.0
+spandx ~>1.0
diff --git a/config/.default-python-packages b/config/.default-python-packages
index 39e3e0a..ddef412 100644
--- a/config/.default-python-packages
+++ b/config/.default-python-packages
@@ -1,2 +1,3 @@
 conan
 pip
+pip-licenses
diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb
index c17ffea..8731e4f 100644
--- a/lib/license/finder/ext.rb
+++ b/lib/license/finder/ext.rb
@@ -3,6 +3,7 @@
 require 'license/finder/ext/license'
 require 'license/finder/ext/maven'
 require 'license/finder/ext/nuget'
+require 'license/finder/ext/pip'
 require 'license/finder/ext/shared_helpers'
 
 # Apply patch to the JsonReport found in the `license_finder` gem.
diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb
new file mode 100644
index 0000000..54b7d40
--- /dev/null
+++ b/lib/license/finder/ext/pip.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+module LicenseFinder
+  class Pip
+    def current_packages
+      detected_dependencies.map do |name, version|
+        PipPackage.new(name, version, pypi.definition_for(name, version))
+      end
+    end
+
+    def possible_package_paths
+      path = project_path || Pathname.pwd
+
+      [
+        path.join(@requirements_path),
+        path.join('setup.py')
+      ]
+    end
+
+    def prepare
+      return install_packages if detected_package_path == @requirements_path
+
+      requirements_path = detected_package_path.dirname.join('requirements.txt')
+      requirements_path.write('.') unless requirements_path.exist?
+      install_packages
+    end
+
+    private
+
+    def detected_dependencies
+      stdout, _stderr, status = execute([
+        python_executable,
+        LicenseFinder::BIN_PATH.join('license_finder_pip.py'),
+        detected_package_path
+      ])
+      return [] unless status.success?
+
+      JSON.parse(stdout).map { |package| package.values_at('name', 'version') }
+    end
+
+    def install_packages
+      execute([prepare_command, "-i", pip_index_url, "-r", @requirements_path])
+    end
+
+    def execute(command)
+      Dir.chdir(project_path) do
+        ::LicenseFinder::SharedHelpers::Cmd.run(Array(command).join(' '))
+      end
+    end
+
+    def python_executable
+      "python#{@python_version == '2' ? '' : '3'}"
+    end
+
+    def pip_index_url
+      ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/')
+    end
+
+    def pypi
+      @pypi ||= Spandx::Python::PyPI.new(sources: [
+        Spandx::Python::Source.new({
+          'name' => 'pypi',
+          'url' => pip_index_url,
+          'verify_ssl' => true
+        })
+      ])
+    end
+  end
+end
diff --git a/lib/license/finder/ext/shared_helpers.rb b/lib/license/finder/ext/shared_helpers.rb
index bc37b9c..b6b6fcd 100644
--- a/lib/license/finder/ext/shared_helpers.rb
+++ b/lib/license/finder/ext/shared_helpers.rb
@@ -4,8 +4,10 @@ module LicenseFinder
   module SharedHelpers
     class Cmd
       def self.run(command)
+        ::License::Management.logger.debug(command)
         stdout, stderr, status = Open3.capture3(command)
-        ::License::Management.logger.debug([command, stdout].join('\n'))
+        ::License::Management.logger.debug(stdout) unless stdout.nil? || stdout.empty?
+        ::License::Management.logger.error(stderr) unless stderr.nil? || stderr.empty?
         [stdout, stderr, status]
       end
     end
diff --git a/lib/license/management.rb b/lib/license/management.rb
index a39c841..16a9d62 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -3,6 +3,7 @@
 require 'json'
 require 'logger'
 require 'pathname'
+require 'spandx'
 require 'yaml'
 
 require 'license_finder'
diff --git a/license-management.gemspec b/license-management.gemspec
index 0372991..e0a25ca 100644
--- a/license-management.gemspec
+++ b/license-management.gemspec
@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_dependency 'license_finder', '~> 6.0.0'
+  spec.add_dependency 'spandx', '~> 0.1'
   spec.add_development_dependency 'gitlab-styles', '~> 3.1'
   spec.add_development_dependency 'json-schema', '~> 2.8'
   spec.add_development_dependency 'rspec', '~> 3.9'
diff --git a/spec/fixtures/build.gradle.kts b/spec/fixtures/java/build.gradle.kts
index 494fc8b..494fc8b 100644
--- a/spec/fixtures/build.gradle.kts
+++ b/spec/fixtures/java/build.gradle.kts
diff --git a/spec/fixtures/custom-maven-settings.xml b/spec/fixtures/java/custom-maven-settings.xml
index 4fa5d16..4fa5d16 100644
--- a/spec/fixtures/custom-maven-settings.xml
+++ b/spec/fixtures/java/custom-maven-settings.xml
diff --git a/spec/fixtures/maven-multimodule/api/pom.xml b/spec/fixtures/java/maven-multimodule/api/pom.xml
index c621c1a..c621c1a 100644
--- a/spec/fixtures/maven-multimodule/api/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/api/pom.xml
diff --git a/spec/fixtures/maven-multimodule/model/pom.xml b/spec/fixtures/java/maven-multimodule/model/pom.xml
index 91b366b..91b366b 100644
--- a/spec/fixtures/maven-multimodule/model/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/model/pom.xml
diff --git a/spec/fixtures/maven-multimodule/pom.xml b/spec/fixtures/java/maven-multimodule/pom.xml
index e84ad4a..e84ad4a 100644
--- a/spec/fixtures/maven-multimodule/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/pom.xml
diff --git a/spec/fixtures/maven-multimodule/web/pom.xml b/spec/fixtures/java/maven-multimodule/web/pom.xml
index 548e9fb..548e9fb 100644
--- a/spec/fixtures/maven-multimodule/web/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/web/pom.xml
diff --git a/spec/fixtures/pom-public-gitlab-repository.xml b/spec/fixtures/java/pom-public-gitlab-repository.xml
index 4e57c79..4e57c79 100644
--- a/spec/fixtures/pom-public-gitlab-repository.xml
+++ b/spec/fixtures/java/pom-public-gitlab-repository.xml
diff --git a/spec/fixtures/drupal_composer.json b/spec/fixtures/php/drupal_composer.json
index 48b445b..48b445b 100644
--- a/spec/fixtures/drupal_composer.json
+++ b/spec/fixtures/php/drupal_composer.json
diff --git a/spec/fixtures/python/complex-setup.py b/spec/fixtures/python/complex-setup.py
new file mode 100644
index 0000000..2478283
--- /dev/null
+++ b/spec/fixtures/python/complex-setup.py
@@ -0,0 +1,213 @@
+"""A setuptools based setup module.
+
+See:
+https://packaging.python.org/guides/distributing-packages-using-setuptools/
+https://github.com/pypa/sampleproject
+"""
+
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+from os import path
+# io.open is needed for projects that support Python 2.7
+# It ensures open() defaults to text mode with universal newlines,
+# and accepts an argument to specify the text encoding
+# Python 3 only projects can skip this import
+from io import open
+
+here = path.abspath(path.dirname(__file__))
+
+# Get the long description from the README file
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+# Arguments marked as "Required" below must be included for upload to PyPI.
+# Fields marked as "Optional" may be commented out.
+
+setup(
+    # This is the name of your project. The first time you publish this
+    # package, this name will be registered for you. It will determine how
+    # users can install this project, e.g.:
+    #
+    # $ pip install sampleproject
+    #
+    # And where it will live on PyPI: https://pypi.org/project/sampleproject/
+    #
+    # There are some restrictions on what makes a valid project name
+    # specification here:
+    # https://packaging.python.org/specifications/core-metadata/#name
+    name='sampleproject',  # Required
+
+    # Versions should comply with PEP 440:
+    # https://www.python.org/dev/peps/pep-0440/
+    #
+    # For a discussion on single-sourcing the version across setup.py and the
+    # project code, see
+    # https://packaging.python.org/en/latest/single_source_version.html
+    version='1.3.1',  # Required
+
+    # This is a one-line description or tagline of what your project does. This
+    # corresponds to the "Summary" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#summary
+    description='A sample Python project',  # Optional
+
+    # This is an optional longer description of your project that represents
+    # the body of text which users will see when they visit PyPI.
+    #
+    # Often, this is the same as your README, so you can just read it in from
+    # that file directly (as we have already done above)
+    #
+    # This field corresponds to the "Description" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#description-optional
+    long_description=long_description,  # Optional
+
+    # Denotes that our long_description is in Markdown; valid values are
+    # text/plain, text/x-rst, and text/markdown
+    #
+    # Optional if long_description is written in reStructuredText (rst) but
+    # required for plain-text or Markdown; if unspecified, "applications should
+    # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and
+    # fall back to text/plain if it is not valid rst" (see link below)
+    #
+    # This field corresponds to the "Description-Content-Type" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional
+    long_description_content_type='text/markdown',  # Optional (see note above)
+
+    # This should be a valid link to your project's main homepage.
+    #
+    # This field corresponds to the "Home-Page" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#home-page-optional
+    url='https://github.com/pypa/sampleproject',  # Optional
+
+    # This should be your name or the name of the organization which owns the
+    # project.
+    author='The Python Packaging Authority',  # Optional
+
+    # This should be a valid email address corresponding to the author listed
+    # above.
+    author_email='pypa-dev@googlegroups.com',  # Optional
+
+    # Classifiers help users find your project by categorizing it.
+    #
+    # For a list of valid classifiers, see https://pypi.org/classifiers/
+    classifiers=[  # Optional
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 3 - Alpha',
+
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Build Tools',
+
+        # Pick your license as you wish
+        'License :: OSI Approved :: MIT License',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        # These classifiers are *not* checked by 'pip install'. See instead
+        # 'python_requires' below.
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+    ],
+
+    # This field adds keywords for your project which will appear on the
+    # project page. What does your project relate to?
+    #
+    # Note that this is a string of words separated by whitespace, not a list.
+    keywords='sample setuptools development',  # Optional
+
+    # When your source code is in a subdirectory under the project root, e.g.
+    # `src/`, it is necessary to specify the `package_dir` argument.
+    package_dir={'': 'src'},  # Optional
+
+    # You can just specify package directories manually here if your project is
+    # simple. Or you can use find_packages().
+    #
+    # Alternatively, if you just want to distribute a single Python file, use
+    # the `py_modules` argument instead as follows, which will expect a file
+    # called `my_module.py` to exist:
+    #
+    #   py_modules=["my_module"],
+    #
+    packages=find_packages(where='src'),  # Required
+
+    # Specify which Python versions you support. In contrast to the
+    # 'Programming Language' classifiers above, 'pip install' will check this
+    # and refuse to install the project if the version does not match. If you
+    # do not support Python 2, you can simplify this to '>=3.5' or similar, see
+    # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4',
+
+    # This field lists other packages that your project depends on to run.
+    # Any package you put here will be installed by pip when your project is
+    # installed, so they must be valid existing projects.
+    #
+    # For an analysis of "install_requires" vs pip's requirements files see:
+    # https://packaging.python.org/en/latest/requirements.html
+    install_requires=['peppercorn'],  # Optional
+
+    # List additional groups of dependencies here (e.g. development
+    # dependencies). Users will be able to install these using the "extras"
+    # syntax, for example:
+    #
+    #   $ pip install sampleproject[dev]
+    #
+    # Similar to `install_requires` above, these must be valid existing
+    # projects.
+    extras_require={  # Optional
+        'dev': ['check-manifest'],
+        'test': ['coverage'],
+    },
+
+    # If there are data files included in your packages that need to be
+    # installed, specify them here.
+    #
+    # If using Python 2.6 or earlier, then these have to be included in
+    # MANIFEST.in as well.
+    package_data={  # Optional
+        'sample': ['package_data.dat'],
+    },
+
+    # Although 'package_data' is the preferred approach, in some case you may
+    # need to place data files outside of your packages. See:
+    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
+    #
+    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
+    data_files=[('my_data', ['data/data_file'])],  # Optional
+
+    # To provide executable scripts, use entry points in preference to the
+    # "scripts" keyword. Entry points provide cross-platform support and allow
+    # `pip` to create the appropriate form of executable for the target
+    # platform.
+    #
+    # For example, the following would provide a command called `sample` which
+    # executes the function `main` from this package when invoked:
+    entry_points={  # Optional
+        'console_scripts': [
+            'sample=sample:main',
+        ],
+    },
+
+    # List additional URLs that are relevant to your project as a dict.
+    #
+    # This field corresponds to the "Project-URL" metadata fields:
+    # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use
+    #
+    # Examples listed include a pattern for specifying where the package tracks
+    # issues, where the source is hosted, where to say thanks to the package
+    # maintainers, and where to support the project financially. The key is
+    # what's used to render the link text on PyPI.
+    project_urls={  # Optional
+        'Bug Reports': 'https://github.com/pypa/sampleproject/issues',
+        'Funding': 'https://donate.pypi.org',
+        'Say Thanks!': 'http://saythanks.io/to/example',
+        'Source': 'https://github.com/pypa/sampleproject/',
+    },
+)
+
diff --git a/spec/fixtures/python/simple-setup.py b/spec/fixtures/python/simple-setup.py
new file mode 100644
index 0000000..02ee1c4
--- /dev/null
+++ b/spec/fixtures/python/simple-setup.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import os
+import shutil
+
+from setuptools import find_packages
+from setuptools import setup
+
+shutil.rmtree("build", ignore_errors=True)
+
+setup(
+    name="package name",
+    version='1.1',
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        "boto3",
+    ],
+    author="author",
+    author_email="author@author.com",
+    description="All the stuff",
+    url="https://www.author.com",
+)
diff --git a/spec/integration/java/gradle_spec.rb b/spec/integration/java/gradle_spec.rb
index 3c63e37..7a510ac 100644
--- a/spec/integration/java/gradle_spec.rb
+++ b/spec/integration/java/gradle_spec.rb
@@ -60,7 +60,7 @@ plugins {
   ].each do |gradle_version|
     %w[8 11].each do |java_version|
       context "when scanning a gradle (v#{gradle_version}) project that uses a kotlin build script" do
-        let(:build_file_content) { fixture_file_content("build.gradle.kts") }
+        let(:build_file_content) { fixture_file_content("java/build.gradle.kts") }
 
         it 'scans a gradle project' do
           runner.add_file('build.gradle.kts', build_file_content)
diff --git a/spec/integration/java/maven_spec.rb b/spec/integration/java/maven_spec.rb
index ad4cf5e..176cb6e 100644
--- a/spec/integration/java/maven_spec.rb
+++ b/spec/integration/java/maven_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe "maven" do
 
   describe "When the maven dependencies come from a custom public maven repository" do
     it 'is able to detect some of the licenses' do
-      runner.add_file('pom.xml', fixture_file_content('pom-public-gitlab-repository.xml'))
+      runner.add_file('pom.xml', fixture_file_content('java/pom-public-gitlab-repository.xml'))
 
       report = runner.scan(env: {
         'CI_PROJECT_ID' => '17523603'
@@ -17,8 +17,8 @@ RSpec.describe "maven" do
     end
 
     it 'downloads packages from by using a custom `settings.xml`' do
-      runner.add_file('pom.xml', fixture_file_content('pom-public-gitlab-repository.xml'))
-      runner.add_file('my_settings.xml', fixture_file_content('custom-maven-settings.xml'))
+      runner.add_file('pom.xml', fixture_file_content('java/pom-public-gitlab-repository.xml'))
+      runner.add_file('my_settings.xml', fixture_file_content('java/custom-maven-settings.xml'))
 
       report = runner.scan(env: {
         'CI_PROJECT_ID' => 'invalid',
@@ -50,7 +50,7 @@ RSpec.describe "maven" do
 
   describe "When scanning a project with multiple modules" do
     before do
-      runner.mount(dir: fixture_file('maven-multimodule'))
+      runner.mount(dir: fixture_file('java/maven-multimodule'))
     end
 
     it 'detects dependences from each module' do
diff --git a/spec/integration/php/composer_spec.rb b/spec/integration/php/composer_spec.rb
index 2b6d697..1419dd4 100644
--- a/spec/integration/php/composer_spec.rb
+++ b/spec/integration/php/composer_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe "composer" do
   context "when the project's dependencies require php-gd e.g. in the case of Drupal" do
     it 'installs the required dependencies and produces a valid report' do
       # composer.json from https://git.drupalcode.org/project/drupal/raw/8.7.x/core/composer.json
-      runner.add_file('composer.json', fixture_file_content('drupal_composer.json'))
+      runner.add_file('composer.json', fixture_file_content('php/drupal_composer.json'))
 
       report = runner.scan
       expect(report).to match_schema(version: '2.0')
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index 9c565a9..e54aa19 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -65,4 +65,37 @@ RSpec.describe "pip" do
       end
     end
   end
+
+  context "when scanning projects with a `setup.py` but do not have a `requirements.txt` files" do
+    pending 'detects licenses in a simple `setup.py`' do
+      runner.add_file('setup.py', fixture_file_content('python/simple-setup.py'))
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).not_to be_empty
+      expect(find_in(report, 'boto3')[:licenses]).to match_array(['MIT'])
+    end
+
+    pending 'detects licenses in a more complicated `setup.py`' do
+      runner.add_file('setup.py', fixture_file_content('python/complex-setup.py'))
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).not_to be_empty
+      expect(find_in(report, 'peppercorn')[:licenses]).to match_array(['BSD-2-Clause'])
+    end
+  end
+
+  context "when scanning projects that have a custom index-url" do
+    before do
+      runner.add_file('requirements.txt', 'pip==18.1')
+    end
+
+    it 'detects the licenses from the custom index' do
+      report = runner.scan(env: { 'PIP_INDEX_URL' => 'https://test.pypi.org/simple/' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(find_in(report, 'pip')[:licenses]).to match_array(["MIT"])
+    end
+  end
 end