Use virtualenv, pip-licenses to scan projects

* Add PIL License to list of normalized licenses
* Update Python 3 v2.0 report fixture
* Add CHANGELOG entry
* Define local variables in bash functions
* Ensure `SETUP_CMD` continues to work for python projects
* build virtualenv app-data cache
* Fallback to legacy scanner when SETUP_CMD is used
* Extract Shell class to be able to pass custom env

25 files changed, 187 insertions, 290 deletions

diff --git a/.gitlab/test.yml b/.gitlab/test.yml
index 6088a87..22147eb 100644
--- a/.gitlab/test.yml
+++ b/.gitlab/test.yml
@@ -2,7 +2,7 @@ size:
   image: docker:stable
   stage: test
   allow_failure: true # temporary until we can shrink the image size.
-  timeout: 1 minute
+  timeout: 3 minutes
   script:
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
     - docker pull $TMP_IMAGE
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 574c667..b6928e4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # GitLab License management changelog
 
+## v3.3.0
+
+- Scan Python projects with [pip-licenses](https://pypi.org/project/pip-licenses/). (!128)
+
 ## v3.2.0
 
 - Install packages from `PIP_INDEX_URL`. (!125)
diff --git a/Dockerfile b/Dockerfile
index 6c0160e..4390330 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -19,9 +19,9 @@ RUN apt-get update -q \
 
 FROM debian:stable-slim as tools-builder
 ENV ASDF_DATA_DIR="/opt/asdf"
-ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:${PATH}"
-ENV TERM="xterm"
 ENV HOME=/root
+ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:${HOME}/.local/bin:${PATH}"
+ENV TERM="xterm"
 WORKDIR $HOME
 COPY config /root
 COPY config/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
diff --git a/Gemfile.lock b/Gemfile.lock
index 8fd6a53..812a1ef 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    license-management (3.2.0)
+    license-management (3.3.0)
       license_finder (~> 6.0.0)
       spandx (~> 0.1)
 
diff --git a/bin/setup b/bin/setup
index da654a1..7b7f9fb 100755
--- a/bin/setup
+++ b/bin/setup
@@ -4,5 +4,5 @@ set -e
 
 cd "$(dirname "$0")/.."
 
-gem install bundler --conservative -v '~> 2.0'
-bundle install
+gem install bundler --conservative -v '~> 2.0' -q
+bundle install --quiet
diff --git a/config/.bashrc b/config/.bashrc
index e85ae56..e883436 100644
--- a/config/.bashrc
+++ b/config/.bashrc
@@ -3,8 +3,8 @@
 alias nuget='mono /usr/local/bin/nuget.exe'
 
 function inflate() {
-  file=$1
-  to_dir=$2
+  local file=$1
+  local to_dir=$2
   if [ -f "$file" ]; then
     echo "Inflating $file in $to_dir"
     tar --use-compress-program zstd -xf "$file" -C "$to_dir"
@@ -13,12 +13,17 @@ function inflate() {
 }
 
 function switch_to() {
-  tool=$1
-  major_version=$2
+  local tool=$1
+  local major_version=$2
+  local version
   version="$(grep "$tool" "$HOME/.tool-versions"| tr ' ' '\n' | grep "^$major_version")"
   asdf shell "$tool" "$version"
 }
 
+function major_version_from() {
+  echo "$1" | cut -d'.' -f1
+}
+
 function enable_dev_mode() {
   unset HISTFILESIZE
   unset HISTSIZE
diff --git a/config/.config/virtualenv/virtualenv.ini b/config/.config/virtualenv/virtualenv.ini
new file mode 100644
index 0000000..208c7bf
--- /dev/null
+++ b/config/.config/virtualenv/virtualenv.ini
@@ -0,0 +1,4 @@
+[virtualenv]
+python = /opt/asdf/shims/python
+activators =
+    bash
diff --git a/config/.default-gems b/config/.default-gems
index 1c3a508..6870122 100644
--- a/config/.default-gems
+++ b/config/.default-gems
@@ -1,4 +1,4 @@
 bundler ~>1.7
 bundler ~>2.0
 license_finder ~>6.0.0
-spandx ~>1.0
+spandx ~>0.1
diff --git a/config/.default-python-packages b/config/.default-python-packages
index ddef412..86cadc4 100644
--- a/config/.default-python-packages
+++ b/config/.default-python-packages
@@ -1,3 +1,3 @@
 conan
 pip
-pip-licenses
+virtualenv
diff --git a/config/install.sh b/config/install.sh
index 0a0ddf2..f015620 100644
--- a/config/install.sh
+++ b/config/install.sh
@@ -97,6 +97,11 @@ asdf install
 asdf reshim
 asdf current
 
+for version in $(asdf list python); do
+  asdf shell python "$version"
+  pip download -d "$HOME/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
+done
+
 rm -fr /tmp
 mkdir -p /tmp
 chmod 777 /tmp
diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb
index 54b7d40..e83f64c 100644
--- a/lib/license/finder/ext/pip.rb
+++ b/lib/license/finder/ext/pip.rb
@@ -3,8 +3,19 @@
 module LicenseFinder
   class Pip
     def current_packages
-      detected_dependencies.map do |name, version|
-        PipPackage.new(name, version, pypi.definition_for(name, version))
+      return legacy_results unless virtual_env?
+
+      _stdout, _stderr, status = pip_licenses
+      return legacy_results unless status.success?
+
+      JSON.parse(IO.read('pip-licenses.json')).map do |dependency|
+        Package.new(
+          dependency['Name'],
+          dependency['Version'],
+          description: dependency['Description'],
+          homepage: dependency['URL'],
+          spec_licenses: [dependency['License']]
+        )
       end
     end
 
@@ -27,35 +38,49 @@ module LicenseFinder
 
     private
 
-    def detected_dependencies
-      stdout, _stderr, status = execute([
-        python_executable,
-        LicenseFinder::BIN_PATH.join('license_finder_pip.py'),
-        detected_package_path
-      ])
-      return [] unless status.success?
-
-      JSON.parse(stdout).map { |package| package.values_at('name', 'version') }
-    end
-
     def install_packages
-      execute([prepare_command, "-i", pip_index_url, "-r", @requirements_path])
+      within_project_dir do
+        shell.execute(['virtualenv -p', python_executable, '--activators=bash --seeder=app-data venv'])
+        shell.sh([". venv/bin/activate", "&&", :pip, :install, '-i', pip_index_url, '-r', @requirements_path])
+      end
     end
 
-    def execute(command)
-      Dir.chdir(project_path) do
-        ::LicenseFinder::SharedHelpers::Cmd.run(Array(command).join(' '))
-      end
+    def pip_licenses
+      shell.sh([
+        ". venv/bin/activate &&",
+        :pip, :install,
+        '--no-index',
+        '--find-links $HOME/.config/virtualenv/app-data', 'pip-licenses', '&&',
+        'pip-licenses',
+        '--ignore-packages prettytable',
+        '--with-description',
+        '--with-urls',
+        '--from=meta',
+        '--format=json',
+        '--output-file pip-licenses.json'
+      ], env: { 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' })
     end
 
     def python_executable
-      "python#{@python_version == '2' ? '' : '3'}"
+      '"$(asdf where python)/bin/python"'
     end
 
     def pip_index_url
       ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/')
     end
 
+    def virtual_env?
+      within_project_dir { File.exist?('venv/bin/activate') }
+    end
+
+    def within_project_dir
+      Dir.chdir(project_path) { yield }
+    end
+
+    def shell
+      @shell ||= ::License::Management::Shell.new
+    end
+
     def pypi
       @pypi ||= Spandx::Python::PyPI.new(sources: [
         Spandx::Python::Source.new({
@@ -65,5 +90,18 @@ module LicenseFinder
         })
       ])
     end
+
+    def legacy_results
+      pip_output.map do |name, version, children, location|
+        spec = pypi.definition_for(name, version)
+        Package.new(
+          name,
+          version,
+          description: spec['description'],
+          homepage: spec['home_page'],
+          spec_licenses: PipPackage.license_names_from_spec(spec)
+        )
+      end
+    end
   end
 end
diff --git a/lib/license/finder/ext/shared_helpers.rb b/lib/license/finder/ext/shared_helpers.rb
index b6b6fcd..cee79ab 100644
--- a/lib/license/finder/ext/shared_helpers.rb
+++ b/lib/license/finder/ext/shared_helpers.rb
@@ -4,11 +4,8 @@ module LicenseFinder
   module SharedHelpers
     class Cmd
       def self.run(command)
-        ::License::Management.logger.debug(command)
-        stdout, stderr, status = Open3.capture3(command)
-        ::License::Management.logger.debug(stdout) unless stdout.nil? || stdout.empty?
-        ::License::Management.logger.error(stderr) unless stderr.nil? || stderr.empty?
-        [stdout, stderr, status]
+        @shell ||= ::License::Management::Shell.new
+        @shell.execute(command)
       end
     end
   end
diff --git a/lib/license/management.rb b/lib/license/management.rb
index 16a9d62..e7a5b23 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -11,6 +11,7 @@ require 'license/management/loggable'
 require 'license/management/verifiable'
 require 'license/management/repository'
 require 'license/management/report'
+require 'license/management/shell'
 require 'license/management/version'
 
 require 'license/finder/ext'
diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb
index 0122018..37bcf37 100644
--- a/lib/license/management/loggable.rb
+++ b/lib/license/management/loggable.rb
@@ -6,14 +6,6 @@ module License
       def logger
         License::Management.logger
       end
-
-      def log_info(message)
-        logger.info(message)
-      end
-
-      def log_error(message)
-        logger.error(message)
-      end
     end
   end
 end
diff --git a/lib/license/management/report/v1.rb b/lib/license/management/report/v1.rb
index 49423c6..27495b5 100644
--- a/lib/license/management/report/v1.rb
+++ b/lib/license/management/report/v1.rb
@@ -31,7 +31,7 @@ module License
           license = { name: join_license_names(dependency.licenses) }
 
           urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort
-          log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1
+          logger.info("multiple urls detected: #{urls.inspect}") if urls.size > 1
           url = urls[0] || license_data(dependency.licenses.first)['url']
 
           license[:url] = url if present?(url)
diff --git a/lib/license/management/report/v2.rb b/lib/license/management/report/v2.rb
index 6ab6b99..f8c96da 100644
--- a/lib/license/management/report/v2.rb
+++ b/lib/license/management/report/v2.rb
@@ -31,7 +31,7 @@ module License
 
         def map_from(dependency)
           licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort
-          log_info [dependency.name, dependency.version, licenses].inspect
+          logger.info [dependency.name, dependency.version, licenses].inspect
 
           {
             name: dependency.name,
diff --git a/lib/license/management/repository.rb b/lib/license/management/repository.rb
index b13cec8..fdd4eae 100644
--- a/lib/license/management/repository.rb
+++ b/lib/license/management/repository.rb
@@ -60,7 +60,7 @@ module License
       end
 
       def generate_item_for(license)
-        log_info("Detected unknown license `#{license.short_name}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.")
+        logger.info("Detected unknown license `#{license.short_name}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.")
         name = take_first_line_from(license.name)
         {
           'id' => name.downcase,
@@ -88,7 +88,7 @@ module License
 
         uri.path.split('/')[-1]
       rescue StandardError => e
-        log_info(e)
+        logger.error(e)
         nil
       end
     end
diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb
new file mode 100644
index 0000000..903d0b6
--- /dev/null
+++ b/lib/license/management/shell.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module License
+  module Management
+    class Shell
+      attr_reader :logger
+
+      def initialize(logger: License::Management.logger)
+        @logger = logger
+      end
+
+      def execute(command, env: {})
+        expanded_command = expand(command)
+        logger.debug(expanded_command)
+
+        stdout, stderr, status = Open3.capture3(env, expanded_command)
+
+        logger.debug(stdout) unless stdout.nil? || stdout.empty?
+        logger.error(stderr) unless stderr.nil? || stderr.empty?
+        [stdout, stderr, status]
+      end
+
+      def sh(command, env: {})
+        execute("sh -c '#{expand(command)}'", env: env)
+      end
+
+      private
+
+      def expand(command)
+        Array(command).map(&:to_s).join(' ')
+      end
+    end
+  end
+end
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
index 946d5e9..22f92ca 100644
--- a/lib/license/management/version.rb
+++ b/lib/license/management/version.rb
@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '3.2.0'
+    VERSION = '3.3.0'
   end
 end
diff --git a/normalized-licenses.yml b/normalized-licenses.yml
index 8b1c643..b78f389 100644
--- a/normalized-licenses.yml
+++ b/normalized-licenses.yml
@@ -6,7 +6,9 @@ ids:
   Apache2: Apache-2.0
   Apache License v2.0: Apache-2.0
   ASL, version 2: Apache-2.0
+  BSD (3 clause): BSD-3-Clause
   BSD: BSD-4-Clause
+  BSD-derived (http://www.repoze.org/LICENSE.txt): BSD-2-Clause
   BSD-like: BSD-4-Clause
   BSD style: BSD-3-Clause
   CC0 1.0 Universal: CC0-1.0
@@ -42,6 +44,7 @@ ids:
   ruby: Ruby
   Simplified BSD: BSD-2-Clause
   SimplifiedBSD: BSD-2-Clause
+  Standard PIL License: MIT-CMU
   The Apache Software License, Version 1.1: Apache-1.1
   The SAX License: SAX-PD
   The W3C License: W3C-20150513
diff --git a/run.sh b/run.sh
index 4020222..efa42b4 100755
--- a/run.sh
+++ b/run.sh
@@ -94,10 +94,6 @@ function prepare_project() {
   fi
 }
 
-function major_version_from() {
-  echo "$1" | cut -d'.' -f1
-}
-
 python_version=$(major_version_from "${LM_PYTHON_VERSION:-3}")
 switch_to python "$python_version"
 switch_to java "adopt-openjdk-${LM_JAVA_VERSION:-8}"
diff --git a/spec/fixtures/expected/python/2/pip/v2.0.json b/spec/fixtures/expected/python/2/pip/v2.0.json
index 1e675ff..8ed6ee7 100644
--- a/spec/fixtures/expected/python/2/pip/v2.0.json
+++ b/spec/fixtures/expected/python/2/pip/v2.0.json
@@ -20,6 +20,12 @@
       "count": 1
     },
     {
+      "id": "MIT-CMU",
+      "name": "CMU License",
+      "url": "https://github.com/python-pillow/Pillow/blob/fffb426092c8db24a5f4b6df243a8a3c01fb63cd/LICENSE",
+      "count": 1
+    },
+    {
       "id": "copyright (c) 2015, julien fache",
       "name": "Copyright (c) 2015, Julien Fache",
       "url": "",
@@ -30,12 +36,6 @@
       "name": "Python License 2.0",
       "url": "https://opensource.org/licenses/Python-2.0",
       "count": 1
-    },
-    {
-      "id": "standard pil license",
-      "name": "Standard PIL License",
-      "url": "",
-      "count": 1
     }
   ],
   "dependencies": [
@@ -58,7 +58,7 @@
         "."
       ],
       "licenses": [
-        "standard pil license"
+        "MIT-CMU"
       ]
     },
     {
diff --git a/spec/fixtures/expected/python/3/pip/v2.0.json b/spec/fixtures/expected/python/3/pip/v2.0.json
index 5d885ee..b5c23ae 100644
--- a/spec/fixtures/expected/python/3/pip/v2.0.json
+++ b/spec/fixtures/expected/python/3/pip/v2.0.json
@@ -26,6 +26,12 @@
       "count": 1
     },
     {
+      "id": "MIT-CMU",
+      "name": "CMU License",
+      "url": "https://github.com/python-pillow/Pillow/blob/fffb426092c8db24a5f4b6df243a8a3c01fb63cd/LICENSE",
+      "count": 1
+    },
+    {
       "id": "copyright (c) 2015, julien fache",
       "name": "Copyright (c) 2015, Julien Fache",
       "url": "",
@@ -36,12 +42,6 @@
       "name": "Python License 2.0",
       "url": "https://opensource.org/licenses/Python-2.0",
       "count": 1
-    },
-    {
-      "id": "standard pil license",
-      "name": "Standard PIL License",
-      "url": "",
-      "count": 1
     }
   ],
   "dependencies": [
@@ -64,7 +64,7 @@
         "."
       ],
       "licenses": [
-        "standard pil license"
+        "MIT-CMU"
       ]
     },
     {
diff --git a/spec/fixtures/python/complex-setup.py b/spec/fixtures/python/complex-setup.py
deleted file mode 100644
index 2478283..0000000
--- a/spec/fixtures/python/complex-setup.py
+++ /dev/null
@@ -1,213 +0,0 @@
-"""A setuptools based setup module.
-
-See:
-https://packaging.python.org/guides/distributing-packages-using-setuptools/
-https://github.com/pypa/sampleproject
-"""
-
-# Always prefer setuptools over distutils
-from setuptools import setup, find_packages
-from os import path
-# io.open is needed for projects that support Python 2.7
-# It ensures open() defaults to text mode with universal newlines,
-# and accepts an argument to specify the text encoding
-# Python 3 only projects can skip this import
-from io import open
-
-here = path.abspath(path.dirname(__file__))
-
-# Get the long description from the README file
-with open(path.join(here, 'README.md'), encoding='utf-8') as f:
-    long_description = f.read()
-
-# Arguments marked as "Required" below must be included for upload to PyPI.
-# Fields marked as "Optional" may be commented out.
-
-setup(
-    # This is the name of your project. The first time you publish this
-    # package, this name will be registered for you. It will determine how
-    # users can install this project, e.g.:
-    #
-    # $ pip install sampleproject
-    #
-    # And where it will live on PyPI: https://pypi.org/project/sampleproject/
-    #
-    # There are some restrictions on what makes a valid project name
-    # specification here:
-    # https://packaging.python.org/specifications/core-metadata/#name
-    name='sampleproject',  # Required
-
-    # Versions should comply with PEP 440:
-    # https://www.python.org/dev/peps/pep-0440/
-    #
-    # For a discussion on single-sourcing the version across setup.py and the
-    # project code, see
-    # https://packaging.python.org/en/latest/single_source_version.html
-    version='1.3.1',  # Required
-
-    # This is a one-line description or tagline of what your project does. This
-    # corresponds to the "Summary" metadata field:
-    # https://packaging.python.org/specifications/core-metadata/#summary
-    description='A sample Python project',  # Optional
-
-    # This is an optional longer description of your project that represents
-    # the body of text which users will see when they visit PyPI.
-    #
-    # Often, this is the same as your README, so you can just read it in from
-    # that file directly (as we have already done above)
-    #
-    # This field corresponds to the "Description" metadata field:
-    # https://packaging.python.org/specifications/core-metadata/#description-optional
-    long_description=long_description,  # Optional
-
-    # Denotes that our long_description is in Markdown; valid values are
-    # text/plain, text/x-rst, and text/markdown
-    #
-    # Optional if long_description is written in reStructuredText (rst) but
-    # required for plain-text or Markdown; if unspecified, "applications should
-    # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and
-    # fall back to text/plain if it is not valid rst" (see link below)
-    #
-    # This field corresponds to the "Description-Content-Type" metadata field:
-    # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional
-    long_description_content_type='text/markdown',  # Optional (see note above)
-
-    # This should be a valid link to your project's main homepage.
-    #
-    # This field corresponds to the "Home-Page" metadata field:
-    # https://packaging.python.org/specifications/core-metadata/#home-page-optional
-    url='https://github.com/pypa/sampleproject',  # Optional
-
-    # This should be your name or the name of the organization which owns the
-    # project.
-    author='The Python Packaging Authority',  # Optional
-
-    # This should be a valid email address corresponding to the author listed
-    # above.
-    author_email='pypa-dev@googlegroups.com',  # Optional
-
-    # Classifiers help users find your project by categorizing it.
-    #
-    # For a list of valid classifiers, see https://pypi.org/classifiers/
-    classifiers=[  # Optional
-        # How mature is this project? Common values are
-        #   3 - Alpha
-        #   4 - Beta
-        #   5 - Production/Stable
-        'Development Status :: 3 - Alpha',
-
-        # Indicate who your project is intended for
-        'Intended Audience :: Developers',
-        'Topic :: Software Development :: Build Tools',
-
-        # Pick your license as you wish
-        'License :: OSI Approved :: MIT License',
-
-        # Specify the Python versions you support here. In particular, ensure
-        # that you indicate whether you support Python 2, Python 3 or both.
-        # These classifiers are *not* checked by 'pip install'. See instead
-        # 'python_requires' below.
-        'Programming Language :: Python :: 2',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Programming Language :: Python :: 3.7',
-        'Programming Language :: Python :: 3.8',
-    ],
-
-    # This field adds keywords for your project which will appear on the
-    # project page. What does your project relate to?
-    #
-    # Note that this is a string of words separated by whitespace, not a list.
-    keywords='sample setuptools development',  # Optional
-
-    # When your source code is in a subdirectory under the project root, e.g.
-    # `src/`, it is necessary to specify the `package_dir` argument.
-    package_dir={'': 'src'},  # Optional
-
-    # You can just specify package directories manually here if your project is
-    # simple. Or you can use find_packages().
-    #
-    # Alternatively, if you just want to distribute a single Python file, use
-    # the `py_modules` argument instead as follows, which will expect a file
-    # called `my_module.py` to exist:
-    #
-    #   py_modules=["my_module"],
-    #
-    packages=find_packages(where='src'),  # Required
-
-    # Specify which Python versions you support. In contrast to the
-    # 'Programming Language' classifiers above, 'pip install' will check this
-    # and refuse to install the project if the version does not match. If you
-    # do not support Python 2, you can simplify this to '>=3.5' or similar, see
-    # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires
-    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4',
-
-    # This field lists other packages that your project depends on to run.
-    # Any package you put here will be installed by pip when your project is
-    # installed, so they must be valid existing projects.
-    #
-    # For an analysis of "install_requires" vs pip's requirements files see:
-    # https://packaging.python.org/en/latest/requirements.html
-    install_requires=['peppercorn'],  # Optional
-
-    # List additional groups of dependencies here (e.g. development
-    # dependencies). Users will be able to install these using the "extras"
-    # syntax, for example:
-    #
-    #   $ pip install sampleproject[dev]
-    #
-    # Similar to `install_requires` above, these must be valid existing
-    # projects.
-    extras_require={  # Optional
-        'dev': ['check-manifest'],
-        'test': ['coverage'],
-    },
-
-    # If there are data files included in your packages that need to be
-    # installed, specify them here.
-    #
-    # If using Python 2.6 or earlier, then these have to be included in
-    # MANIFEST.in as well.
-    package_data={  # Optional
-        'sample': ['package_data.dat'],
-    },
-
-    # Although 'package_data' is the preferred approach, in some case you may
-    # need to place data files outside of your packages. See:
-    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
-    #
-    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
-    data_files=[('my_data', ['data/data_file'])],  # Optional
-
-    # To provide executable scripts, use entry points in preference to the
-    # "scripts" keyword. Entry points provide cross-platform support and allow
-    # `pip` to create the appropriate form of executable for the target
-    # platform.
-    #
-    # For example, the following would provide a command called `sample` which
-    # executes the function `main` from this package when invoked:
-    entry_points={  # Optional
-        'console_scripts': [
-            'sample=sample:main',
-        ],
-    },
-
-    # List additional URLs that are relevant to your project as a dict.
-    #
-    # This field corresponds to the "Project-URL" metadata fields:
-    # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use
-    #
-    # Examples listed include a pattern for specifying where the package tracks
-    # issues, where the source is hosted, where to say thanks to the package
-    # maintainers, and where to support the project financially. The key is
-    # what's used to render the link text on PyPI.
-    project_urls={  # Optional
-        'Bug Reports': 'https://github.com/pypa/sampleproject/issues',
-        'Funding': 'https://donate.pypi.org',
-        'Say Thanks!': 'http://saythanks.io/to/example',
-        'Source': 'https://github.com/pypa/sampleproject/',
-    },
-)
-
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index e54aa19..d22121b 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe "pip" do
       expect(report).to match_schema(version: '2.0')
       expect(report[:version]).to start_with('2')
       expect(report[:dependencies].map { |x| x[:name] }).to include("sentry-sdk")
-      expect(report[:dependencies].find { |x| x[:name] == 'sentry-sdk' }[:licenses]).to match_array(["BSD-4-Clause"])
+      expect(find_in(report, 'sentry-sdk')[:licenses]).to match_array(["BSD-4-Clause"])
     end
   end
 
@@ -54,30 +54,31 @@ RSpec.describe "pip" do
         let(:language) { 'python' }
         let(:package_manager) { 'pip' }
         let(:environment) { { 'LM_REPORT_VERSION' => report_version, 'LM_PYTHON_VERSION' => python[:version] } }
+        let(:expected_content) { fixture_file_content("expected/#{language}/#{python[:version]}/#{package_manager}/v#{report_version}.json").chomp }
 
         it 'matches the expected report' do
           runner.clone(url, branch: python[:commit])
           report = runner.scan(env: environment)
-          content = fixture_file_content("expected/#{language}/#{python[:version]}/#{package_manager}/v#{report_version}.json")
-          expect(report).to eq(JSON.parse(content, symbolize_names: true))
+
+          expect(JSON.pretty_generate(report)).to eq(expected_content)
           expect(report).to match_schema(version: report_version)
         end
       end
     end
   end
 
-  context "when scanning projects with a `setup.py` but do not have a `requirements.txt` files" do
-    pending 'detects licenses in a simple `setup.py`' do
+  context "when scanning projects with a `setup.py` and does not have a `requirements.txt` file" do
+    it 'detects licenses in a simple `setup.py`' do
       runner.add_file('setup.py', fixture_file_content('python/simple-setup.py'))
       report = runner.scan
 
       expect(report).to match_schema(version: '2.0')
       expect(report[:dependencies]).not_to be_empty
-      expect(find_in(report, 'boto3')[:licenses]).to match_array(['MIT'])
+      expect(find_in(report, 'boto3')[:licenses]).to match_array(['Apache-2.0'])
     end
 
-    pending 'detects licenses in a more complicated `setup.py`' do
-      runner.add_file('setup.py', fixture_file_content('python/complex-setup.py'))
+    it 'detects licenses in a more complicated `setup.py`' do
+      runner.clone('https://github.com/pypa/sampleproject.git', branch: 'd09af3dbd851d385e56f0aed29875bfa3d3df230')
       report = runner.scan
 
       expect(report).to match_schema(version: '2.0')
@@ -88,14 +89,44 @@ RSpec.describe "pip" do
 
   context "when scanning projects that have a custom index-url" do
     before do
-      runner.add_file('requirements.txt', 'pip==18.1')
+      runner.add_file('requirements.txt', 'six')
     end
 
     it 'detects the licenses from the custom index' do
       report = runner.scan(env: { 'PIP_INDEX_URL' => 'https://test.pypi.org/simple/' })
 
       expect(report).to match_schema(version: '2.0')
-      expect(find_in(report, 'pip')[:licenses]).to match_array(["MIT"])
+      expect(find_in(report, 'six')[:licenses]).to match_array(["MIT"])
+    end
+  end
+
+  context "when a project uses a custom `SETUP_CMD`" do
+    before do
+      runner.add_file('requirements.txt', 'six==1.14.0')
+    end
+
+    it 'detects the software licenses' do
+      report = runner.scan(env: { 'SETUP_CMD' => 'pip install -r requirements.txt' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(find_in(report, 'six')[:licenses]).to match_array(["MIT"])
+      expect(report[:dependencies].map { |x| x[:name] }).to contain_exactly('six')
+    end
+  end
+
+  context "when a projects is running in airgap mode" do
+    before do
+      runner.add_file('requirements.txt', '')
+    end
+
+    it 'is able to scan the project' do
+      report = runner.scan(env: {
+        'PIP_INDEX_URL' => 'https://localhost/simple/'
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:licenses]).to be_empty
+      expect(report[:dependencies]).to be_empty
     end
   end
 end