Use virtualenv, pip-licenses to scan projects

* Add PIL License to list of normalized licenses
* Update Python 3 v2.0 report fixture
* Add CHANGELOG entry
* Define local variables in bash functions
* Ensure `SETUP_CMD` continues to work for python projects
* build virtualenv app-data cache
* Fallback to legacy scanner when SETUP_CMD is used
* Extract Shell class to be able to pass custom env

1 files changed, 41 insertions, 10 deletions

diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index e54aa19..d22121b 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -13,7 +13,7 @@ RSpec.describe "pip" do
       expect(report).to match_schema(version: '2.0')
       expect(report[:version]).to start_with('2')
       expect(report[:dependencies].map { |x| x[:name] }).to include("sentry-sdk")
-      expect(report[:dependencies].find { |x| x[:name] == 'sentry-sdk' }[:licenses]).to match_array(["BSD-4-Clause"])
+      expect(find_in(report, 'sentry-sdk')[:licenses]).to match_array(["BSD-4-Clause"])
     end
   end
 
@@ -54,30 +54,31 @@ RSpec.describe "pip" do
         let(:language) { 'python' }
         let(:package_manager) { 'pip' }
         let(:environment) { { 'LM_REPORT_VERSION' => report_version, 'LM_PYTHON_VERSION' => python[:version] } }
+        let(:expected_content) { fixture_file_content("expected/#{language}/#{python[:version]}/#{package_manager}/v#{report_version}.json").chomp }
 
         it 'matches the expected report' do
           runner.clone(url, branch: python[:commit])
           report = runner.scan(env: environment)
-          content = fixture_file_content("expected/#{language}/#{python[:version]}/#{package_manager}/v#{report_version}.json")
-          expect(report).to eq(JSON.parse(content, symbolize_names: true))
+
+          expect(JSON.pretty_generate(report)).to eq(expected_content)
           expect(report).to match_schema(version: report_version)
         end
       end
     end
   end
 
-  context "when scanning projects with a `setup.py` but do not have a `requirements.txt` files" do
-    pending 'detects licenses in a simple `setup.py`' do
+  context "when scanning projects with a `setup.py` and does not have a `requirements.txt` file" do
+    it 'detects licenses in a simple `setup.py`' do
       runner.add_file('setup.py', fixture_file_content('python/simple-setup.py'))
       report = runner.scan
 
       expect(report).to match_schema(version: '2.0')
       expect(report[:dependencies]).not_to be_empty
-      expect(find_in(report, 'boto3')[:licenses]).to match_array(['MIT'])
+      expect(find_in(report, 'boto3')[:licenses]).to match_array(['Apache-2.0'])
     end
 
-    pending 'detects licenses in a more complicated `setup.py`' do
-      runner.add_file('setup.py', fixture_file_content('python/complex-setup.py'))
+    it 'detects licenses in a more complicated `setup.py`' do
+      runner.clone('https://github.com/pypa/sampleproject.git', branch: 'd09af3dbd851d385e56f0aed29875bfa3d3df230')
       report = runner.scan
 
       expect(report).to match_schema(version: '2.0')
@@ -88,14 +89,44 @@ RSpec.describe "pip" do
 
   context "when scanning projects that have a custom index-url" do
     before do
-      runner.add_file('requirements.txt', 'pip==18.1')
+      runner.add_file('requirements.txt', 'six')
     end
 
     it 'detects the licenses from the custom index' do
       report = runner.scan(env: { 'PIP_INDEX_URL' => 'https://test.pypi.org/simple/' })
 
       expect(report).to match_schema(version: '2.0')
-      expect(find_in(report, 'pip')[:licenses]).to match_array(["MIT"])
+      expect(find_in(report, 'six')[:licenses]).to match_array(["MIT"])
+    end
+  end
+
+  context "when a project uses a custom `SETUP_CMD`" do
+    before do
+      runner.add_file('requirements.txt', 'six==1.14.0')
+    end
+
+    it 'detects the software licenses' do
+      report = runner.scan(env: { 'SETUP_CMD' => 'pip install -r requirements.txt' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(find_in(report, 'six')[:licenses]).to match_array(["MIT"])
+      expect(report[:dependencies].map { |x| x[:name] }).to contain_exactly('six')
+    end
+  end
+
+  context "when a projects is running in airgap mode" do
+    before do
+      runner.add_file('requirements.txt', '')
+    end
+
+    it 'is able to scan the project' do
+      report = runner.scan(env: {
+        'PIP_INDEX_URL' => 'https://localhost/simple/'
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:licenses]).to be_empty
+      expect(report[:dependencies]).to be_empty
     end
   end
 end