2 files changed, 384 insertions, 0 deletions
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
new file mode 100644
index 0000000..8e3ec3d
--- /dev/null
+++ b/spec/integration/python/pip_spec.rb
@@ -0,0 +1,160 @@
+require 'spec_helper'
+
+RSpec.describe "pip" do
+  context "when a project depends on the latest version of pip" do
+    let(:requirements) { "sentry-sdk>=0.7.7" }
+
+    it 'produces a valid report' do
+      runner.add_file('requirements.txt', requirements)
+
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:version]).to start_with('2')
+      expect(report.dependency_names).to include("sentry-sdk")
+      expect(report.licenses_for('sentry-sdk')).to match_array(["BSD-4-Clause"])
+    end
+  end
+
+  context "when the project has a dependency that depends on a minimum of python 3.6" do
+    let(:requirements) do
+      [
+        'boto3',
+        'aws-lambda-context>=1.0.0',
+        'jsonschema>=3.0.0',
+        'python-json-logger>=0.1.10',
+        'sentry-sdk>=0.7.7',
+        'ptvsd',
+        'pylint',
+        'flake8',
+        'bandit',
+        'pydocstyle'
+      ].join("\n")
+    end
+
+    it 'produces a valid report' do
+      runner.add_file('requirements.txt', requirements)
+
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:version]).to start_with('2')
+      expect(report[:licenses]).not_to be_empty
+      expect(report[:dependencies]).not_to be_empty
+    end
+  end
+
+  [{ version: '2', commit: '04dce91b' }, { version: '3', commit: '48e250a1' }].each do |python|
+    ['1.0', '1.1', '2.0'].each do |report_version|
+      context "when generating a `#{report_version}` report using Python `#{python[:version]}`" do
+        let(:url) { "https://gitlab.com/gitlab-org/security-products/tests/#{language}-#{package_manager}.git" }
+        let(:language) { 'python' }
+        let(:package_manager) { 'pip' }
+        let(:environment) { { 'LM_REPORT_VERSION' => report_version, 'LM_PYTHON_VERSION' => python[:version] } }
+        let(:expected_content) { fixture_file_content("expected/#{language}/#{python[:version]}/#{package_manager}/v#{report_version}.json").chomp }
+
+        it 'matches the expected report' do
+          runner.clone(url, branch: python[:commit])
+          report = runner.scan(env: environment)
+
+          expect(JSON.pretty_generate(report.to_h)).to eq(expected_content)
+          expect(report).to match_schema(version: report_version)
+        end
+      end
+    end
+  end
+
+  context "when scanning projects with a `setup.py` and does not have a `requirements.txt` file" do
+    it 'detects licenses in a simple `setup.py`' do
+      runner.add_file('setup.py', fixture_file_content('python/simple-setup.py'))
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).not_to be_empty
+      expect(report.licenses_for('boto3')).to match_array(['Apache-2.0'])
+    end
+
+    it 'detects licenses in a more complicated `setup.py`' do
+      runner.clone('https://github.com/pypa/sampleproject.git', branch: 'd09af3dbd851d385e56f0aed29875bfa3d3df230')
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).not_to be_empty
+      expect(report.licenses_for('peppercorn')).to match_array(['BSD-2-Clause'])
+    end
+  end
+
+  context "when scanning projects that have a custom index-url" do
+    before do
+      runner.add_file('requirements.txt', 'six')
+    end
+
+    it 'detects the licenses from the custom index' do
+      report = runner.scan(env: { 'PIP_INDEX_URL' => 'https://test.pypi.org/simple/' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report.licenses_for('six')).to match_array(["MIT"])
+    end
+  end
+
+  context "when a project uses a custom `SETUP_CMD`" do
+    before do
+      runner.add_file('requirements.txt', 'six==1.14.0')
+    end
+
+    it 'detects the software licenses' do
+      report = runner.scan(env: { 'SETUP_CMD' => 'pip install -r requirements.txt' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report.licenses_for('six')).to match_array(["MIT"])
+      expect(report.dependency_names).to contain_exactly('six')
+    end
+  end
+
+  context "when a projects is running in airgap mode" do
+    before do
+      runner.add_file('requirements.txt', '')
+    end
+
+    it 'is able to scan the project' do
+      report = runner.scan(env: {
+        'PIP_INDEX_URL' => 'https://localhost/simple/'
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:licenses]).to be_empty
+      expect(report[:dependencies]).to be_empty
+    end
+  end
+
+  context "when connecting to a private package repository with self signed certificate" do
+    let(:index_url) { "https://#{private_pypi_host}/simple" }
+    let(:bundle) { fixture_file_content('python/pypi.crt') }
+
+    before do
+      runner.add_file('setup.py') do
+        <<~RAW
+from setuptools import setup, find_packages
+
+setup(
+    name='gitlab-sp-test-python-pip',
+    version='1.2.0',
+    packages=find_packages(),
+    install_requires=['requests'],
+)
+        RAW
+      end
+    end
+
+    it 'downloads the packages and trusts the certificate' do
+      report = runner.scan(env: {
+        'ADDITIONAL_CA_CERT_BUNDLE' => bundle,
+        'PIP_INDEX_URL' => index_url
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report.dependency_names).to include('requests')
+      expect(report.licenses_for('requests')).to match_array(['Apache-2.0'])
+    end
+  end
+end
diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb
new file mode 100644
index 0000000..b54ee99
--- /dev/null
+++ b/spec/integration/python/pipenv_spec.rb
@@ -0,0 +1,224 @@
+require 'spec_helper'
+
+RSpec.describe "pipenv" do
+  include_examples "each report version", "python", "pipenv", "pip-file-lock"
+
+  context "when a project depends on a version 6 Pipfile.lock" do
+    let(:pipfile_lock_content) do
+      JSON.pretty_generate({
+        "_meta": {
+          "hash": { "sha256": "" },
+          "pipfile-spec": 6,
+          "requires": { "python_version": "3.8" },
+          "sources": [{ "name": "pypi", "url": "https://pypi.org/simple", "verify_ssl": true }]
+        },
+        "default": {
+          "six": { "hashes": [], "index": "pypi", "version": "==1.13.0" }
+        },
+        "develop": {}
+      })
+    end
+
+    it 'produces a valid report' do
+      runner.add_file('Pipfile.lock', pipfile_lock_content)
+
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:version]).not_to be_empty
+      expect(report[:licenses]).not_to be_empty
+      expect(report.dependency_names).to contain_exactly("six")
+    end
+  end
+
+  context "when a project depends on a version 3.2.1 Pipfile.lock" do
+    let(:pipfile_lock_content) do
+      JSON.pretty_generate({
+        "default": {
+          "crayons": { "version": "==0.1.2", "hash": "" },
+          "requirements-parser": { "version": "==0.1.0", "hash": "" },
+          "pexpect": { "version": "==4.2.1", "hash": "" },
+          "delegator.py": { "version": "==0.0.8", "hash": "" },
+          "backports.shutil_get_terminal_size": { "version": "==1.0.0", "hash": "" },
+          "ptyprocess": { "version": "==0.5.1", "hash": "" },
+          "parse": { "version": "==1.6.6", "hash": "" },
+          "toml": { "version": "==0.9.2", "hash": "" },
+          "colorama": { "version": "==0.3.7", "hash": "" },
+          "requests": { "version": "==2.13.0", "hash": "" },
+          "click": { "version": "==6.7", "hash": "" }
+        },
+        "develop": {
+          "packaging": { "version": "==16.8", "hash": "" },
+          "pytest": { "version": "==3.0.6", "hash": "" },
+          "setuptools": { "version": "==34.0.2", "hash": "" },
+          "pyparsing": { "version": "==2.1.10", "hash": "" },
+          "py": { "version": "==1.4.32", "hash": "" },
+          "six": { "version": "==1.10.0", "hash": "" },
+          "appdirs": { "version": "==1.4.0", "hash": "" }
+        },
+        "_meta": {
+          "sources": [{ "url": "https://pypi.python.org/simple", "verify_ssl": true }],
+          "requires": {},
+          "Pipfile-sha256": "24f12b631b7c40b8c5eff934a1aef263ed04f5eaffb4acf4706442f3d23cba36"
+        }
+      })
+    end
+
+    it 'produces a valid report' do
+      runner.add_file('Pipfile.lock', pipfile_lock_content)
+
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:version]).not_to be_empty
+      expect(report[:licenses]).not_to be_empty
+      expect(report.dependency_names).to match_array([
+        "backports.shutil_get_terminal_size",
+        "click",
+        "colorama",
+        "crayons",
+        "delegator.py",
+        "parse",
+        "pexpect",
+        "ptyprocess",
+        "requests",
+        "requirements-parser",
+        "toml"
+      ])
+    end
+  end
+
+  context "when a project depends on a version 5 Pipfile.lock" do
+    let(:pipfile_lock_content) do
+      JSON.pretty_generate({
+        "_meta": {
+          "hash": { "sha256": "" },
+          "host-environment-markers": {
+            "implementation_name": "cpython",
+            "implementation_version": "3.6.1",
+            "os_name": "posix",
+            "platform_machine": "x86_64",
+            "platform_python_implementation": "CPython",
+            "platform_release": "16.7.0",
+            "platform_system": "Darwin",
+            "platform_version": "Darwin Kernel Version 16.7.0: Thu Jun 15 17:36:27 PDT 2017; root:xnu-3789.70.16~2/RELEASE_X86_64",
+            "python_full_version": "3.6.1",
+            "python_version": "3.6",
+            "sys_platform": "darwin"
+          },
+          "pipfile-spec": 5,
+          "requires": {},
+          "sources": [{ "name": "pypi", "url": "https://pypi.python.org/simple", "verify_ssl": true }]
+        },
+        "default": {
+          "certifi": { "hashes": ["", ""], "version": "==2017.7.27.1" },
+          "chardet": { "hashes": ["", ""], "version": "==3.0.4" },
+          "idna": { "hashes": ["", ""], "version": "==2.6" },
+          "requests": { "hashes": ["", ""], "version": "==2.18.4" },
+          "urllib3": { "hashes": ["", ""], "version": "==1.22" }
+        },
+        "develop": {
+          "py": { "hashes": ["", ""], "version": "==1.4.34" },
+          "pytest": { "hashes": ["", ""], "version": "==3.2.2" }
+        }
+      })
+    end
+
+    it 'produces a valid report' do
+      runner.add_file('Pipfile.lock', pipfile_lock_content)
+
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:version]).not_to be_empty
+      expect(report[:licenses]).not_to be_empty
+      expect(report.dependency_names).to match_array(%w[
+        certifi
+        chardet
+        idna
+        requests
+        urllib3
+      ])
+    end
+  end
+
+  context "when fetching metadata from a custom source" do
+    let(:pipfile_lock_content) do
+      JSON.pretty_generate({
+        "_meta": {
+          "hash": { "sha256": "" },
+          "pipfile-spec": 6,
+          "requires": { "python_version": "3.8" },
+          "sources": [{ "name": "pypi", "url": "https://test.pypi.org/simple", "verify_ssl": true }]
+        },
+        "default": {
+          "six": { "hashes": [], "index": "pypi", "version": "==1.13.0" }
+        },
+        "develop": {}
+      })
+    end
+
+    before do
+      runner.add_file('Pipfile.lock', pipfile_lock_content)
+    end
+
+    it 'produces a valid report' do
+      report = runner.scan
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:licenses]).not_to be_empty
+      expect(report[:dependencies].count).to be(1)
+      expect(report.find('six')).not_to be_nil
+    end
+  end
+
+  context "when scanning a simple Pipfile project" do
+    let(:lockfile_content) { fixture_file_content('python/simple-Pipfile.lock') }
+    let(:lockfile_hash) { JSON.parse(lockfile_content) }
+
+    before do
+      runner.add_file('Pipfile', fixture_file_content('python/simple-Pipfile'))
+      runner.add_file('Pipfile.lock', lockfile_content)
+    end
+
+    [2, 3].each do |version|
+      context "when scanning a Python #{version} project" do
+        let(:report) { runner.scan(env: { 'LM_PYTHON_VERSION' => version.to_s }) }
+
+        specify { expect(report).to match_schema(version: '2.0') }
+
+        it 'includes dependencies in the default group' do
+          lockfile_hash['default'].keys.each do |key|
+            expect(report.find(key)).not_to be_nil
+          end
+        end
+
+        it 'excludes dependencies in the development group' do
+          lockfile_hash['develop'].keys.each do |key|
+            expect(report.find(key)).to be_nil
+          end
+        end
+      end
+    end
+  end
+
+  context "when connecting to a private package repository with self signed certificate" do
+    let(:index_url) { "https://#{private_pypi_host}/simple" }
+    let(:bundle) { fixture_file_content('python/pypi.crt') }
+
+    before do
+      runner.add_file('Pipfile', fixture_file_content('python/airgap-Pipfile.erb', index_url: index_url))
+      runner.add_file('Pipfile.lock', fixture_file_content('python/airgap-Pipfile.lock.erb', index_url: index_url))
+    end
+
+    it 'downloads the packages and trusts the certificate' do
+      report = runner.scan(env: {
+        'ADDITIONAL_CA_CERT_BUNDLE' => bundle,
+        'PIP_INDEX_URL' => index_url
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report.dependency_names).to include('requests')
+    end
+  end
+end