Merge branch '10128-go-modules' into 'master'v3.5.0

Improve license detection in go modules projects

See merge request gitlab-org/security-products/license-management!129

9 files changed, 75 insertions, 35 deletions

diff --git a/spec/integration/dotnet/examples_spec.rb b/spec/integration/dotnet/examples_spec.rb
index 87459ee..13f4712 100644
--- a/spec/integration/dotnet/examples_spec.rb
+++ b/spec/integration/dotnet/examples_spec.rb
@@ -5,7 +5,6 @@ RSpec.describe ".NET Core" do
     runner.clone('https://github.com/microsoft/RockPaperScissorsLizardSpock.git')
     report = runner.scan(env: { 'LICENSE_FINDER_CLI_OPTS' => '--recursive' })
 
-    expect(report).not_to be_empty
     expect(report).to match_schema(version: '2.0')
     expect(report[:licenses].count).not_to be_zero
     expect(report[:dependencies].count).not_to be_zero
diff --git a/spec/integration/dotnet/nuget_spec.rb b/spec/integration/dotnet/nuget_spec.rb
index 0efbcf0..6eeb261 100644
--- a/spec/integration/dotnet/nuget_spec.rb
+++ b/spec/integration/dotnet/nuget_spec.rb
@@ -26,10 +26,9 @@ RSpec.describe "nuget" do
       runner.add_file('packages.config', packages_config)
       report = runner.scan
 
-      expect(report).not_to be_empty
       expect(report).to match_schema(version: '2.0')
       expect(report[:licenses].count).not_to be_zero
-      expect(report[:dependencies].map { |x| x[:name] }).to match_array([
+      expect(report.dependency_names).to match_array([
         "Microsoft.CodeDom.Providers.DotNetCompilerPlatform",
         "Microsoft.Net.Compilers",
         "Microsoft.Web.Infrastructure",
@@ -68,10 +67,9 @@ RSpec.describe "nuget" do
       runner.add_file('winforms/packages.config', winforms_packages)
       report = runner.scan(env: { 'LICENSE_FINDER_CLI_OPTS' => '--recursive' })
 
-      expect(report).not_to be_empty
       expect(report).to match_schema(version: '2.0')
-      expect(find_in(report, 'jive')[:licenses]).to match_array(['MIT'])
-      expect(find_in(report, 'MvcMailer')[:licenses]).to match_array(['MIT'])
+      expect(report.licenses_for('jive')).to match_array(['MIT'])
+      expect(report.licenses_for('MvcMailer')).to match_array(['MIT'])
     end
   end
 
@@ -89,11 +87,10 @@ RSpec.describe "nuget" do
       runner.add_file('packages.config', packages)
       report = runner.scan
 
-      expect(report).not_to be_empty
       expect(report).to match_schema(version: '2.0')
       expect(report[:licenses].count).not_to be_zero
       expect(report[:licenses].map { |x| x[:id] }.uniq).to match_array(['LGPL-2.1', 'Apache-2.0', 'BSD-3-Clause'])
-      expect(report[:dependencies].map { |x| x[:name] }).to match_array([
+      expect(report.dependency_names).to match_array([
         'Iesi.Collections',
         'Remotion.Linq',
         'Remotion.Linq.EagerFetching',
diff --git a/spec/integration/go/modules_spec.rb b/spec/integration/go/modules_spec.rb
index ebe0359..143d8ea 100644
--- a/spec/integration/go/modules_spec.rb
+++ b/spec/integration/go/modules_spec.rb
@@ -2,4 +2,52 @@ require 'spec_helper'
 
 RSpec.describe "modules" do
   include_examples "each report version", "go", "modules"
+
+  context "when scanning a customers go.mod and go.sum files" do
+    let(:report) { runner.scan }
+
+    before do
+      runner.add_file('main.go', fixture_file_content('go/main.go'))
+      runner.add_file('go.mod', fixture_file_content('go/go.mod'))
+      runner.add_file('go.sum', fixture_file_content('go/go.sum'))
+    end
+
+    specify { expect(report).to match_schema(version: '2.0') }
+    specify { expect(report[:licenses]).not_to be_empty }
+
+    specify do
+      expect(report.dependency_names).to match_array([
+        "github.com/davecgh/go-spew",
+        "github.com/dimfeld/httptreemux/v5",
+        "github.com/go-logfmt/logfmt",
+        "github.com/golang/protobuf",
+        "github.com/google/uuid",
+        "github.com/pmezard/go-difflib",
+        "github.com/stretchr/objx",
+        "golang.org/x/net",
+        "golang.org/x/oauth2",
+        "google.golang.org/appengine",
+        "gopkg.in/yaml.v2",
+        'github.com/stretchr/testify'
+      ])
+    end
+
+    specify { expect(report.licenses_for('github.com/dimfeld/httptreemux/v5')).to match_array(['MIT']) }
+    specify { expect(report.licenses_for('github.com/go-logfmt/logfmt')).to match_array(['MIT']) }
+    specify { expect(report.licenses_for('github.com/google/uuid')).to match_array(['BSD-3-Clause']) }
+    specify { expect(report.licenses_for('github.com/stretchr/testify')).to match_array(['MIT']) }
+    specify { expect(report.licenses_for('golang.org/x/oauth2')).to match_array(['BSD-3-Clause']) }
+  end
+
+  context "when scanning the `gitaly` project" do
+    let(:report) { runner.scan }
+
+    before do
+      runner.clone('https://gitlab.com/gitlab-org/gitaly.git')
+    end
+
+    specify { expect(report).to match_schema(version: '2.0') }
+    specify { expect(report[:licenses]).not_to be_empty }
+    specify { expect(report[:dependencies]).not_to be_empty }
+  end
 end
diff --git a/spec/integration/java/gradle_spec.rb b/spec/integration/java/gradle_spec.rb
index 7a510ac..1b566bb 100644
--- a/spec/integration/java/gradle_spec.rb
+++ b/spec/integration/java/gradle_spec.rb
@@ -34,7 +34,6 @@ plugins {
     end
 
     it 'is able to detect licenses' do
-      expect(result).not_to be_empty
       expect(result).to match_schema(version: '2.0')
       expect(result[:licenses]).not_to be_empty
 
@@ -45,7 +44,7 @@ plugins {
         { name: 'sitemesh', licenses: ['Apache-1.1'] },
         { name: 'hibernate-jpa-2.1-api', licenses: ['BSD-3-Clause', 'EPL-1.0'] }
       ].each do |dependency|
-        expect(find_in(result, dependency[:name])[:licenses]).to match_array(dependency[:licenses])
+        expect(result.licenses_for(dependency[:name])).to match_array(dependency[:licenses])
       end
     end
   end
diff --git a/spec/integration/java/maven_spec.rb b/spec/integration/java/maven_spec.rb
index 176cb6e..92444e8 100644
--- a/spec/integration/java/maven_spec.rb
+++ b/spec/integration/java/maven_spec.rb
@@ -72,10 +72,10 @@ RSpec.describe "maven" do
         { name: "netty-all", licenses: ["Apache-2.0"] },
         { name: "stax2-api", licenses: ["BSD-4-Clause"] }
       ].each do |dependency|
-        expect(find_in(report, dependency[:name])[:licenses]).to match_array(dependency[:licenses])
+        expect(report.licenses_for(dependency[:name])).to match_array(dependency[:licenses])
       end
 
-      expect(report[:dependencies].map { |x| x[:name] }).not_to include('junit')
+      expect(report.dependency_names).not_to include('junit')
     end
   end
 end
diff --git a/spec/integration/php/composer_spec.rb b/spec/integration/php/composer_spec.rb
index 1419dd4..62ff598 100644
--- a/spec/integration/php/composer_spec.rb
+++ b/spec/integration/php/composer_spec.rb
@@ -9,10 +9,11 @@ RSpec.describe "composer" do
       runner.add_file('composer.json', fixture_file_content('php/drupal_composer.json'))
 
       report = runner.scan
+
       expect(report).to match_schema(version: '2.0')
       expect(report[:version]).not_to be_empty
       expect(report[:licenses]).not_to be_empty
-      expect(report[:dependencies].map { |x| x[:name] }).to match_array(%w[
+      expect(report.dependency_names).to match_array(%w[
         asm89/stack-cors
         behat/mink
         behat/mink-browserkit-driver
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index d22121b..2183dd1 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -9,11 +9,10 @@ RSpec.describe "pip" do
 
       report = runner.scan
 
-      expect(report).not_to be_empty
       expect(report).to match_schema(version: '2.0')
       expect(report[:version]).to start_with('2')
-      expect(report[:dependencies].map { |x| x[:name] }).to include("sentry-sdk")
-      expect(find_in(report, 'sentry-sdk')[:licenses]).to match_array(["BSD-4-Clause"])
+      expect(report.dependency_names).to include("sentry-sdk")
+      expect(report.licenses_for('sentry-sdk')).to match_array(["BSD-4-Clause"])
     end
   end
 
@@ -39,7 +38,6 @@ RSpec.describe "pip" do
 
       report = runner.scan
 
-      expect(report).not_to be_empty
       expect(report).to match_schema(version: '2.0')
       expect(report[:version]).to start_with('2')
       expect(report[:licenses]).not_to be_empty
@@ -60,7 +58,7 @@ RSpec.describe "pip" do
           runner.clone(url, branch: python[:commit])
           report = runner.scan(env: environment)
 
-          expect(JSON.pretty_generate(report)).to eq(expected_content)
+          expect(JSON.pretty_generate(report.to_h)).to eq(expected_content)
           expect(report).to match_schema(version: report_version)
         end
       end
@@ -74,7 +72,7 @@ RSpec.describe "pip" do
 
       expect(report).to match_schema(version: '2.0')
       expect(report[:dependencies]).not_to be_empty
-      expect(find_in(report, 'boto3')[:licenses]).to match_array(['Apache-2.0'])
+      expect(report.licenses_for('boto3')).to match_array(['Apache-2.0'])
     end
 
     it 'detects licenses in a more complicated `setup.py`' do
@@ -83,7 +81,7 @@ RSpec.describe "pip" do
 
       expect(report).to match_schema(version: '2.0')
       expect(report[:dependencies]).not_to be_empty
-      expect(find_in(report, 'peppercorn')[:licenses]).to match_array(['BSD-2-Clause'])
+      expect(report.licenses_for('peppercorn')).to match_array(['BSD-2-Clause'])
     end
   end
 
@@ -96,7 +94,7 @@ RSpec.describe "pip" do
       report = runner.scan(env: { 'PIP_INDEX_URL' => 'https://test.pypi.org/simple/' })
 
       expect(report).to match_schema(version: '2.0')
-      expect(find_in(report, 'six')[:licenses]).to match_array(["MIT"])
+      expect(report.licenses_for('six')).to match_array(["MIT"])
     end
   end
 
@@ -109,8 +107,8 @@ RSpec.describe "pip" do
       report = runner.scan(env: { 'SETUP_CMD' => 'pip install -r requirements.txt' })
 
       expect(report).to match_schema(version: '2.0')
-      expect(find_in(report, 'six')[:licenses]).to match_array(["MIT"])
-      expect(report[:dependencies].map { |x| x[:name] }).to contain_exactly('six')
+      expect(report.licenses_for('six')).to match_array(["MIT"])
+      expect(report.dependency_names).to contain_exactly('six')
     end
   end
 
diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb
index 983ea8b..410d3c7 100644
--- a/spec/integration/python/pipenv_spec.rb
+++ b/spec/integration/python/pipenv_spec.rb
@@ -24,11 +24,10 @@ RSpec.describe "pipenv" do
 
       report = runner.scan
 
-      expect(report).not_to be_empty
       expect(report).to match_schema(version: '2.0')
       expect(report[:version]).not_to be_empty
       expect(report[:licenses]).not_to be_empty
-      expect(report[:dependencies].map { |x| x[:name] }).to contain_exactly("six")
+      expect(report.dependency_names).to contain_exactly("six")
     end
   end
 
@@ -71,10 +70,9 @@ RSpec.describe "pipenv" do
       report = runner.scan
 
       expect(report).to match_schema(version: '2.0')
-      expect(report).not_to be_empty
       expect(report[:version]).not_to be_empty
       expect(report[:licenses]).not_to be_empty
-      expect(report[:dependencies].map { |x| x[:name] }).to match_array([
+      expect(report.dependency_names).to match_array([
         "backports.shutil_get_terminal_size",
         "click",
         "colorama",
@@ -132,10 +130,9 @@ RSpec.describe "pipenv" do
       report = runner.scan
 
       expect(report).to match_schema(version: '2.0')
-      expect(report).not_to be_empty
       expect(report[:version]).not_to be_empty
       expect(report[:licenses]).not_to be_empty
-      expect(report[:dependencies].map { |x| x[:name] }).to match_array(%w[
+      expect(report.dependency_names).to match_array(%w[
         certifi
         chardet
         idna
@@ -171,7 +168,7 @@ RSpec.describe "pipenv" do
       expect(report).to match_schema(version: '2.0')
       expect(report[:licenses]).not_to be_empty
       expect(report[:dependencies].count).to be(1)
-      expect(find_in(report, 'six')).not_to be_nil
+      expect(report.find('six')).not_to be_nil
     end
   end
 
@@ -192,13 +189,13 @@ RSpec.describe "pipenv" do
 
         it 'includes dependencies in the default group' do
           lockfile_hash['default'].keys.each do |key|
-            expect(find_in(report, key)).not_to be_nil
+            expect(report.find(key)).not_to be_nil
           end
         end
 
         it 'excludes dependencies in the development group' do
           lockfile_hash['develop'].keys.each do |key|
-            expect(find_in(report, key)).to be_nil
+            expect(report.find(key)).to be_nil
           end
         end
       end
diff --git a/spec/integration/ruby/bundler_spec.rb b/spec/integration/ruby/bundler_spec.rb
index f2f2873..5236adf 100644
--- a/spec/integration/ruby/bundler_spec.rb
+++ b/spec/integration/ruby/bundler_spec.rb
@@ -82,10 +82,10 @@ BUNDLED WITH
       end
 
       report = runner.scan
-      expect(report).not_to be_empty
+
       expect(report).to match_schema(version: '2.0')
       expect(report[:licenses]).not_to be_empty
-      expect(report[:dependencies].map { |x| x[:name] }).to include("saml-kit")
+      expect(report.dependency_names).to include("saml-kit")
     end
   end
 
@@ -117,9 +117,10 @@ BUNDLED WITH
       end
 
       report = runner.scan
+
       expect(report).to match_schema(version: '2.0')
       expect(report[:licenses]).not_to be_empty
-      expect(find_in(report, 'net-hippie')).to eql({
+      expect(report.find('net-hippie')).to eql({
         name: 'net-hippie',
         description: "net/http for hippies. ☮️",
         url: "https://github.com/mokhan/net-hippie/",