Merge branch '199059-setup-py' into 'master'v3.2.0

Read `PIP_INDEX_URL` to install python packages.

See merge request gitlab-org/security-products/license-management!125

14 files changed, 274 insertions, 6 deletions

diff --git a/spec/fixtures/build.gradle.kts b/spec/fixtures/java/build.gradle.kts
index 494fc8b..494fc8b 100644
--- a/spec/fixtures/build.gradle.kts
+++ b/spec/fixtures/java/build.gradle.kts
diff --git a/spec/fixtures/custom-maven-settings.xml b/spec/fixtures/java/custom-maven-settings.xml
index 4fa5d16..4fa5d16 100644
--- a/spec/fixtures/custom-maven-settings.xml
+++ b/spec/fixtures/java/custom-maven-settings.xml
diff --git a/spec/fixtures/maven-multimodule/api/pom.xml b/spec/fixtures/java/maven-multimodule/api/pom.xml
index c621c1a..c621c1a 100644
--- a/spec/fixtures/maven-multimodule/api/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/api/pom.xml
diff --git a/spec/fixtures/maven-multimodule/model/pom.xml b/spec/fixtures/java/maven-multimodule/model/pom.xml
index 91b366b..91b366b 100644
--- a/spec/fixtures/maven-multimodule/model/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/model/pom.xml
diff --git a/spec/fixtures/maven-multimodule/pom.xml b/spec/fixtures/java/maven-multimodule/pom.xml
index e84ad4a..e84ad4a 100644
--- a/spec/fixtures/maven-multimodule/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/pom.xml
diff --git a/spec/fixtures/maven-multimodule/web/pom.xml b/spec/fixtures/java/maven-multimodule/web/pom.xml
index 548e9fb..548e9fb 100644
--- a/spec/fixtures/maven-multimodule/web/pom.xml
+++ b/spec/fixtures/java/maven-multimodule/web/pom.xml
diff --git a/spec/fixtures/pom-public-gitlab-repository.xml b/spec/fixtures/java/pom-public-gitlab-repository.xml
index 4e57c79..4e57c79 100644
--- a/spec/fixtures/pom-public-gitlab-repository.xml
+++ b/spec/fixtures/java/pom-public-gitlab-repository.xml
diff --git a/spec/fixtures/drupal_composer.json b/spec/fixtures/php/drupal_composer.json
index 48b445b..48b445b 100644
--- a/spec/fixtures/drupal_composer.json
+++ b/spec/fixtures/php/drupal_composer.json
diff --git a/spec/fixtures/python/complex-setup.py b/spec/fixtures/python/complex-setup.py
new file mode 100644
index 0000000..2478283
--- /dev/null
+++ b/spec/fixtures/python/complex-setup.py
@@ -0,0 +1,213 @@
+"""A setuptools based setup module.
+
+See:
+https://packaging.python.org/guides/distributing-packages-using-setuptools/
+https://github.com/pypa/sampleproject
+"""
+
+# Always prefer setuptools over distutils
+from setuptools import setup, find_packages
+from os import path
+# io.open is needed for projects that support Python 2.7
+# It ensures open() defaults to text mode with universal newlines,
+# and accepts an argument to specify the text encoding
+# Python 3 only projects can skip this import
+from io import open
+
+here = path.abspath(path.dirname(__file__))
+
+# Get the long description from the README file
+with open(path.join(here, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
+# Arguments marked as "Required" below must be included for upload to PyPI.
+# Fields marked as "Optional" may be commented out.
+
+setup(
+    # This is the name of your project. The first time you publish this
+    # package, this name will be registered for you. It will determine how
+    # users can install this project, e.g.:
+    #
+    # $ pip install sampleproject
+    #
+    # And where it will live on PyPI: https://pypi.org/project/sampleproject/
+    #
+    # There are some restrictions on what makes a valid project name
+    # specification here:
+    # https://packaging.python.org/specifications/core-metadata/#name
+    name='sampleproject',  # Required
+
+    # Versions should comply with PEP 440:
+    # https://www.python.org/dev/peps/pep-0440/
+    #
+    # For a discussion on single-sourcing the version across setup.py and the
+    # project code, see
+    # https://packaging.python.org/en/latest/single_source_version.html
+    version='1.3.1',  # Required
+
+    # This is a one-line description or tagline of what your project does. This
+    # corresponds to the "Summary" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#summary
+    description='A sample Python project',  # Optional
+
+    # This is an optional longer description of your project that represents
+    # the body of text which users will see when they visit PyPI.
+    #
+    # Often, this is the same as your README, so you can just read it in from
+    # that file directly (as we have already done above)
+    #
+    # This field corresponds to the "Description" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#description-optional
+    long_description=long_description,  # Optional
+
+    # Denotes that our long_description is in Markdown; valid values are
+    # text/plain, text/x-rst, and text/markdown
+    #
+    # Optional if long_description is written in reStructuredText (rst) but
+    # required for plain-text or Markdown; if unspecified, "applications should
+    # attempt to render [the long_description] as text/x-rst; charset=UTF-8 and
+    # fall back to text/plain if it is not valid rst" (see link below)
+    #
+    # This field corresponds to the "Description-Content-Type" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#description-content-type-optional
+    long_description_content_type='text/markdown',  # Optional (see note above)
+
+    # This should be a valid link to your project's main homepage.
+    #
+    # This field corresponds to the "Home-Page" metadata field:
+    # https://packaging.python.org/specifications/core-metadata/#home-page-optional
+    url='https://github.com/pypa/sampleproject',  # Optional
+
+    # This should be your name or the name of the organization which owns the
+    # project.
+    author='The Python Packaging Authority',  # Optional
+
+    # This should be a valid email address corresponding to the author listed
+    # above.
+    author_email='pypa-dev@googlegroups.com',  # Optional
+
+    # Classifiers help users find your project by categorizing it.
+    #
+    # For a list of valid classifiers, see https://pypi.org/classifiers/
+    classifiers=[  # Optional
+        # How mature is this project? Common values are
+        #   3 - Alpha
+        #   4 - Beta
+        #   5 - Production/Stable
+        'Development Status :: 3 - Alpha',
+
+        # Indicate who your project is intended for
+        'Intended Audience :: Developers',
+        'Topic :: Software Development :: Build Tools',
+
+        # Pick your license as you wish
+        'License :: OSI Approved :: MIT License',
+
+        # Specify the Python versions you support here. In particular, ensure
+        # that you indicate whether you support Python 2, Python 3 or both.
+        # These classifiers are *not* checked by 'pip install'. See instead
+        # 'python_requires' below.
+        'Programming Language :: Python :: 2',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Programming Language :: Python :: 3.7',
+        'Programming Language :: Python :: 3.8',
+    ],
+
+    # This field adds keywords for your project which will appear on the
+    # project page. What does your project relate to?
+    #
+    # Note that this is a string of words separated by whitespace, not a list.
+    keywords='sample setuptools development',  # Optional
+
+    # When your source code is in a subdirectory under the project root, e.g.
+    # `src/`, it is necessary to specify the `package_dir` argument.
+    package_dir={'': 'src'},  # Optional
+
+    # You can just specify package directories manually here if your project is
+    # simple. Or you can use find_packages().
+    #
+    # Alternatively, if you just want to distribute a single Python file, use
+    # the `py_modules` argument instead as follows, which will expect a file
+    # called `my_module.py` to exist:
+    #
+    #   py_modules=["my_module"],
+    #
+    packages=find_packages(where='src'),  # Required
+
+    # Specify which Python versions you support. In contrast to the
+    # 'Programming Language' classifiers above, 'pip install' will check this
+    # and refuse to install the project if the version does not match. If you
+    # do not support Python 2, you can simplify this to '>=3.5' or similar, see
+    # https://packaging.python.org/guides/distributing-packages-using-setuptools/#python-requires
+    python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, <4',
+
+    # This field lists other packages that your project depends on to run.
+    # Any package you put here will be installed by pip when your project is
+    # installed, so they must be valid existing projects.
+    #
+    # For an analysis of "install_requires" vs pip's requirements files see:
+    # https://packaging.python.org/en/latest/requirements.html
+    install_requires=['peppercorn'],  # Optional
+
+    # List additional groups of dependencies here (e.g. development
+    # dependencies). Users will be able to install these using the "extras"
+    # syntax, for example:
+    #
+    #   $ pip install sampleproject[dev]
+    #
+    # Similar to `install_requires` above, these must be valid existing
+    # projects.
+    extras_require={  # Optional
+        'dev': ['check-manifest'],
+        'test': ['coverage'],
+    },
+
+    # If there are data files included in your packages that need to be
+    # installed, specify them here.
+    #
+    # If using Python 2.6 or earlier, then these have to be included in
+    # MANIFEST.in as well.
+    package_data={  # Optional
+        'sample': ['package_data.dat'],
+    },
+
+    # Although 'package_data' is the preferred approach, in some case you may
+    # need to place data files outside of your packages. See:
+    # http://docs.python.org/3.4/distutils/setupscript.html#installing-additional-files
+    #
+    # In this case, 'data_file' will be installed into '<sys.prefix>/my_data'
+    data_files=[('my_data', ['data/data_file'])],  # Optional
+
+    # To provide executable scripts, use entry points in preference to the
+    # "scripts" keyword. Entry points provide cross-platform support and allow
+    # `pip` to create the appropriate form of executable for the target
+    # platform.
+    #
+    # For example, the following would provide a command called `sample` which
+    # executes the function `main` from this package when invoked:
+    entry_points={  # Optional
+        'console_scripts': [
+            'sample=sample:main',
+        ],
+    },
+
+    # List additional URLs that are relevant to your project as a dict.
+    #
+    # This field corresponds to the "Project-URL" metadata fields:
+    # https://packaging.python.org/specifications/core-metadata/#project-url-multiple-use
+    #
+    # Examples listed include a pattern for specifying where the package tracks
+    # issues, where the source is hosted, where to say thanks to the package
+    # maintainers, and where to support the project financially. The key is
+    # what's used to render the link text on PyPI.
+    project_urls={  # Optional
+        'Bug Reports': 'https://github.com/pypa/sampleproject/issues',
+        'Funding': 'https://donate.pypi.org',
+        'Say Thanks!': 'http://saythanks.io/to/example',
+        'Source': 'https://github.com/pypa/sampleproject/',
+    },
+)
+
diff --git a/spec/fixtures/python/simple-setup.py b/spec/fixtures/python/simple-setup.py
new file mode 100644
index 0000000..02ee1c4
--- /dev/null
+++ b/spec/fixtures/python/simple-setup.py
@@ -0,0 +1,22 @@
+#!/usr/bin/env python3
+import os
+import shutil
+
+from setuptools import find_packages
+from setuptools import setup
+
+shutil.rmtree("build", ignore_errors=True)
+
+setup(
+    name="package name",
+    version='1.1',
+    packages=find_packages(),
+    include_package_data=True,
+    install_requires=[
+        "boto3",
+    ],
+    author="author",
+    author_email="author@author.com",
+    description="All the stuff",
+    url="https://www.author.com",
+)
diff --git a/spec/integration/java/gradle_spec.rb b/spec/integration/java/gradle_spec.rb
index 3c63e37..7a510ac 100644
--- a/spec/integration/java/gradle_spec.rb
+++ b/spec/integration/java/gradle_spec.rb
@@ -60,7 +60,7 @@ plugins {
   ].each do |gradle_version|
     %w[8 11].each do |java_version|
       context "when scanning a gradle (v#{gradle_version}) project that uses a kotlin build script" do
-        let(:build_file_content) { fixture_file_content("build.gradle.kts") }
+        let(:build_file_content) { fixture_file_content("java/build.gradle.kts") }
 
         it 'scans a gradle project' do
           runner.add_file('build.gradle.kts', build_file_content)
diff --git a/spec/integration/java/maven_spec.rb b/spec/integration/java/maven_spec.rb
index ad4cf5e..176cb6e 100644
--- a/spec/integration/java/maven_spec.rb
+++ b/spec/integration/java/maven_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe "maven" do
 
   describe "When the maven dependencies come from a custom public maven repository" do
     it 'is able to detect some of the licenses' do
-      runner.add_file('pom.xml', fixture_file_content('pom-public-gitlab-repository.xml'))
+      runner.add_file('pom.xml', fixture_file_content('java/pom-public-gitlab-repository.xml'))
 
       report = runner.scan(env: {
         'CI_PROJECT_ID' => '17523603'
@@ -17,8 +17,8 @@ RSpec.describe "maven" do
     end
 
     it 'downloads packages from by using a custom `settings.xml`' do
-      runner.add_file('pom.xml', fixture_file_content('pom-public-gitlab-repository.xml'))
-      runner.add_file('my_settings.xml', fixture_file_content('custom-maven-settings.xml'))
+      runner.add_file('pom.xml', fixture_file_content('java/pom-public-gitlab-repository.xml'))
+      runner.add_file('my_settings.xml', fixture_file_content('java/custom-maven-settings.xml'))
 
       report = runner.scan(env: {
         'CI_PROJECT_ID' => 'invalid',
@@ -50,7 +50,7 @@ RSpec.describe "maven" do
 
   describe "When scanning a project with multiple modules" do
     before do
-      runner.mount(dir: fixture_file('maven-multimodule'))
+      runner.mount(dir: fixture_file('java/maven-multimodule'))
     end
 
     it 'detects dependences from each module' do
diff --git a/spec/integration/php/composer_spec.rb b/spec/integration/php/composer_spec.rb
index 2b6d697..1419dd4 100644
--- a/spec/integration/php/composer_spec.rb
+++ b/spec/integration/php/composer_spec.rb
@@ -6,7 +6,7 @@ RSpec.describe "composer" do
   context "when the project's dependencies require php-gd e.g. in the case of Drupal" do
     it 'installs the required dependencies and produces a valid report' do
       # composer.json from https://git.drupalcode.org/project/drupal/raw/8.7.x/core/composer.json
-      runner.add_file('composer.json', fixture_file_content('drupal_composer.json'))
+      runner.add_file('composer.json', fixture_file_content('php/drupal_composer.json'))
 
       report = runner.scan
       expect(report).to match_schema(version: '2.0')
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index 9c565a9..e54aa19 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -65,4 +65,37 @@ RSpec.describe "pip" do
       end
     end
   end
+
+  context "when scanning projects with a `setup.py` but do not have a `requirements.txt` files" do
+    pending 'detects licenses in a simple `setup.py`' do
+      runner.add_file('setup.py', fixture_file_content('python/simple-setup.py'))
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).not_to be_empty
+      expect(find_in(report, 'boto3')[:licenses]).to match_array(['MIT'])
+    end
+
+    pending 'detects licenses in a more complicated `setup.py`' do
+      runner.add_file('setup.py', fixture_file_content('python/complex-setup.py'))
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).not_to be_empty
+      expect(find_in(report, 'peppercorn')[:licenses]).to match_array(['BSD-2-Clause'])
+    end
+  end
+
+  context "when scanning projects that have a custom index-url" do
+    before do
+      runner.add_file('requirements.txt', 'pip==18.1')
+    end
+
+    it 'detects the licenses from the custom index' do
+      report = runner.scan(env: { 'PIP_INDEX_URL' => 'https://test.pypi.org/simple/' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(find_in(report, 'pip')[:licenses]).to match_array(["MIT"])
+    end
+  end
 end