Merge branch 'offline-tests' into 'master'v3.19.2

Combine default/custom x509 certificates in TLS validation

See merge request gitlab-org/security-products/license-management!194

17 files changed, 131 insertions, 57 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 19f155b..0e75026 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # GitLab License management changelog
 
+## v3.19.2
+
+- Combine default/custom x509 certificates in TLS validation. (!194)
+
 ## v3.19.1
 
 - Choose a version of the `org.codehaus.mojo:license-maven-plugin:aggregate-download-licenses` that is compatible with the version of Maven used by the project. (!195)
diff --git a/Gemfile.lock b/Gemfile.lock
index 12f0d02..907b440 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -8,7 +8,7 @@ GIT
 PATH
   remote: .
   specs:
-    license-management (3.19.1)
+    license-management (3.19.2)
       license_finder (~> 6.6.0)
 
 GEM
diff --git a/bin/test b/bin/test
index a05c7e7..210cebc 100755
--- a/bin/test
+++ b/bin/test
@@ -10,4 +10,4 @@ export RUBYLIB="/builds/gitlab-org/security-products/license-management/lib"
 if ! command -v bundle; then
   ./bin/setup
 fi
-bundle exec rspec "$@" --format=progress --tag ~environment:offline
+bundle exec rspec "$@" --format=progress
diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb
index 2219b18..9b302f6 100644
--- a/lib/license/finder/ext/bower.rb
+++ b/lib/license/finder/ext/bower.rb
@@ -3,7 +3,15 @@
 module LicenseFinder
   class Bower < PackageManager
     def prepare
-      shell.execute([:bower, :install, '--allow-root'], env: default_env)
+      shell.execute([
+        :bower,
+        :install,
+        '--allow-root',
+        '--production',
+        '--verbose',
+        '--loglevel',
+        :debug
+      ], env: default_env)
     end
 
     def current_packages
@@ -18,7 +26,7 @@ module LicenseFinder
 
     def bower_output
       stdout, _stderr, status = Dir.chdir(project_path) do
-        shell.execute([:bower, :list, '--json', '-l', 'action', '--allow-root'])
+        shell.execute([:bower, :list, '--json', '-l', 'action', '--allow-root'], env: default_env)
       end
       return {} unless status.success?
 
@@ -48,12 +56,15 @@ module LicenseFinder
     end
 
     def default_env
-      return {} unless shell.custom_certificate_installed?
-
-      {
-        'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.custom_certificate_path.to_s),
-        'bower_ca' => ENV.fetch('bower_ca', shell.custom_certificate_path.to_s)
+      @default_env ||= {
+        'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.default_certificate_path).to_s,
+        'bower_ca' => ENV.fetch('bower_ca', shell.default_certificate_path).to_s,
+        'bower_directory' => ENV.fetch('bower_directory', vendor_path.join('bower_components')).to_s
       }
     end
+
+    def vendor_path
+      Pathname.pwd.join('.gitlab', 'cache', 'vendor')
+    end
   end
 end
diff --git a/lib/license/management/python.rb b/lib/license/management/python.rb
index 740b5c4..ede792e 100644
--- a/lib/license/management/python.rb
+++ b/lib/license/management/python.rb
@@ -54,9 +54,7 @@ module License
       end
 
       def default_env
-        return {} unless shell.custom_certificate_installed?
-
-        { 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.custom_certificate_path.to_s) }
+        { 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s }
       end
     end
   end
diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb
index 47639df..1a99895 100644
--- a/lib/license/management/shell.rb
+++ b/lib/license/management/shell.rb
@@ -4,11 +4,12 @@ module License
   module Management
     class Shell
       SPLIT_SCRIPT = "'BEGIN {x=0;} /BEGIN CERT/{x++} { print > \"custom.\" x \".crt\" }'"
-      attr_reader :custom_certificate_path, :logger
+      attr_reader :default_certificate_path, :custom_certificate_path, :logger
 
       def initialize(logger: License::Management.logger, certificate: ENV['ADDITIONAL_CA_CERT_BUNDLE'])
         @logger = logger
         @custom_certificate_path = Pathname.new('/usr/local/share/ca-certificates/custom.crt')
+        @default_certificate_path = Pathname.new('/etc/ssl/certs/ca-certificates.crt')
         trust!(certificate) if present?(certificate)
       end
 
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
index 5402857..d9e2383 100644
--- a/lib/license/management/version.rb
+++ b/lib/license/management/version.rb
@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '3.19.1'
+    VERSION = '3.19.2'
   end
 end
diff --git a/spec/fixtures/expected/js/bower/v1.0.json b/spec/fixtures/expected/js/bower/v1.0.json
index f71e6b5..9f1ea46 100644
--- a/spec/fixtures/expected/js/bower/v1.0.json
+++ b/spec/fixtures/expected/js/bower/v1.0.json
@@ -5,16 +5,16 @@
       "name": "MIT"
     },
     {
-      "count": 1,
-      "name": "ISC"
+      "count": 2,
+      "name": "unknown"
     },
     {
       "count": 1,
-      "name": "Simplified BSD, New BSD, MIT, ISC, Apache 2.0"
+      "name": "ISC"
     },
     {
       "count": 1,
-      "name": "unknown"
+      "name": "Simplified BSD, New BSD, MIT, ISC, Apache 2.0"
     }
   ],
   "dependencies": [
@@ -34,6 +34,19 @@
     },
     {
       "license": {
+        "name": "unknown"
+      },
+      "dependency": {
+        "name": "debug",
+        "url": "https://github.com/wixo/debug",
+        "description": "",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "license": {
         "name": "MIT",
         "url": "http://opensource.org/licenses/mit-license"
       },
diff --git a/spec/fixtures/expected/js/bower/v1.1.json b/spec/fixtures/expected/js/bower/v1.1.json
index 9abb729..9058bb5 100644
--- a/spec/fixtures/expected/js/bower/v1.1.json
+++ b/spec/fixtures/expected/js/bower/v1.1.json
@@ -6,16 +6,16 @@
       "name": "MIT"
     },
     {
-      "count": 1,
-      "name": "ISC"
+      "count": 2,
+      "name": "unknown"
     },
     {
       "count": 1,
-      "name": "Simplified BSD, New BSD, MIT, ISC, Apache 2.0"
+      "name": "ISC"
     },
     {
       "count": 1,
-      "name": "unknown"
+      "name": "Simplified BSD, New BSD, MIT, ISC, Apache 2.0"
     }
   ],
   "dependencies": [
@@ -58,6 +58,25 @@
     {
       "licenses": [
         {
+          "name": "unknown",
+          "url": ""
+        }
+      ],
+      "license": {
+        "name": "unknown"
+      },
+      "dependency": {
+        "name": "debug",
+        "url": "https://github.com/wixo/debug",
+        "description": "",
+        "pathes": [
+          "."
+        ]
+      }
+    },
+    {
+      "licenses": [
+        {
           "name": "MIT",
           "url": "http://opensource.org/licenses/mit-license"
         }
diff --git a/spec/fixtures/expected/js/bower/v2.0.json b/spec/fixtures/expected/js/bower/v2.0.json
index 8d62873..050178c 100644
--- a/spec/fixtures/expected/js/bower/v2.0.json
+++ b/spec/fixtures/expected/js/bower/v2.0.json
@@ -14,6 +14,12 @@
       "count": 2
     },
     {
+      "id": "unknown",
+      "name": "unknown",
+      "url": "",
+      "count": 2
+    },
+    {
       "id": "Apache-2.0",
       "name": "Apache License 2.0",
       "url": "https://opensource.org/licenses/Apache-2.0",
@@ -30,12 +36,6 @@
       "name": "BSD 3-Clause \"New\" or \"Revised\" License",
       "url": "https://opensource.org/licenses/BSD-3-Clause",
       "count": 1
-    },
-    {
-      "id": "unknown",
-      "name": "unknown",
-      "url": "",
-      "count": 1
     }
   ],
   "dependencies": [
@@ -55,6 +55,17 @@
       ]
     },
     {
+      "name": "debug",
+      "url": "https://github.com/wixo/debug",
+      "description": "",
+      "paths": [
+        "."
+      ],
+      "licenses": [
+        "unknown"
+      ]
+    },
+    {
       "name": "ev-emitter",
       "url": "https://github.com/metafizzy/ev-emitter",
       "description": "lil' event emitter",
diff --git a/spec/fixtures/expected/js/bower/v2.1.json b/spec/fixtures/expected/js/bower/v2.1.json
index ab5fe6d..78a4a05 100644
--- a/spec/fixtures/expected/js/bower/v2.1.json
+++ b/spec/fixtures/expected/js/bower/v2.1.json
@@ -47,6 +47,15 @@
       ]
     },
     {
+      "name": "debug",
+      "version": "*",
+      "package_manager": "bower",
+      "path": "bower.json",
+      "licenses": [
+        "unknown"
+      ]
+    },
+    {
       "name": "ev-emitter",
       "version": "1.1.1",
       "package_manager": "bower",
@@ -128,4 +137,4 @@
       ]
     }
   ]
-}
+}
\ No newline at end of file
diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb
index 6d51394..227718e 100644
--- a/spec/integration/js/bower_spec.rb
+++ b/spec/integration/js/bower_spec.rb
@@ -11,7 +11,7 @@ RSpec.describe "bower" do
     end
 
     specify { expect(subject).to match_schema }
-    specify { expect(subject.dependency_names).to match_array(['cli', 'ev-emitter', 'example-project', 'fizzy-ui-utils', 'get-size', 'jquery', 'masonry-layout', 'matches-selector', 'outlayer', 'stimulus.umd']) }
+    specify { expect(subject.dependency_names).to match_array(['debug', 'cli', 'ev-emitter', 'example-project', 'fizzy-ui-utils', 'get-size', 'jquery', 'masonry-layout', 'matches-selector', 'outlayer', 'stimulus.umd']) }
     specify { expect(subject.licenses_for('cli')).to match_array(['Apache-2.0', 'BSD-2-Clause', 'BSD-3-Clause', 'ISC', 'MIT']) }
     specify { expect(subject.licenses_for('ev-emitter')).to match_array(['MIT']) }
     specify { expect(subject.licenses_for('example-project')).to match_array(['ISC']) }
@@ -24,10 +24,15 @@ RSpec.describe "bower" do
     specify { expect(subject.licenses_for('stimulus.umd')).to match_array(['unknown']) }
   end
 
-  context "when scanning a bower project with a dependency from a custom npm registry", environment: 'offline' do
-    subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => fixture_file_content('js/custom-npm.crt') }) }
+  context "when scanning a bower project with a dependency from a custom npm registry" do
+    subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read }) }
+
+    let(:private_npm_host) { 'npm.test' }
 
     before do
+      add_host(private_npm_host, '127.0.0.1')
+      start_proxy_server
+
       runner.add_file(".npmrc", "registry = https://#{private_npm_host}")
       runner.add_file(".bowerrc") do
         JSON.pretty_generate({ resolvers: ['bower-npm-resolver'] })
@@ -42,7 +47,8 @@ RSpec.describe "bower" do
     end
 
     specify { expect(subject).to match_schema }
-    specify { expect(subject.dependency_names).to match_array(%w[js-bower lodash]) }
+    specify { expect(subject.dependency_names).to match_array(%w[debug js-bower lodash]) }
+    specify { expect(subject.licenses_for('debug')).to match_array(['unknown']) }
     specify { expect(subject.licenses_for('js-bower')).to match_array(['ISC']) }
     specify { expect(subject.licenses_for('lodash')).to match_array(['MIT']) }
   end
diff --git a/spec/integration/js/npm_spec.rb b/spec/integration/js/npm_spec.rb
index 0efc97a..a8dd2d4 100644
--- a/spec/integration/js/npm_spec.rb
+++ b/spec/integration/js/npm_spec.rb
@@ -467,10 +467,15 @@ RSpec.describe "npm" do
     end
   end
 
-  context "when scanning a project with dependencies sourced from a custom registry", environment: 'offline' do
-    subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => fixture_file_content('js/custom-npm.crt') }) }
+  context "when scanning a project with dependencies sourced from a custom registry" do
+    subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read }) }
+
+    let(:private_npm_host) { 'npm.test' }
 
     before do
+      add_host('npm.test', '127.0.0.1')
+      start_proxy_server
+
       runner.add_file(".npmrc", "registry = https://#{private_npm_host}")
       runner.add_file("package.json") do
         JSON.pretty_generate({
diff --git a/spec/integration/js/yarn_spec.rb b/spec/integration/js/yarn_spec.rb
index bacac47..d81564c 100644
--- a/spec/integration/js/yarn_spec.rb
+++ b/spec/integration/js/yarn_spec.rb
@@ -150,10 +150,15 @@ RSpec.describe "yarn" do
     end
   end
 
-  context "when scanning a project with dependencies sourced from a custom registry", environment: 'offline' do
-    subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => fixture_file_content('js/custom-npm.crt') }) }
+  context "when scanning a project with dependencies sourced from a custom registry" do
+    subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read }) }
+
+    let(:private_npm_host) { 'npm.test' }
 
     before do
+      add_host('npm.test', '127.0.0.1')
+      start_proxy_server
+
       runner.add_file(".yarnrc.yml", "npmRegistryServer: \"https://#{private_npm_host}\"")
 
       runner.add_file("package.json") do
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index e808188..a52b992 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -127,11 +127,13 @@ RSpec.describe "pip" do
     end
   end
 
-  context "when connecting to a private package repository with self signed certificate", environment: 'offline' do
-    let(:index_url) { "https://#{private_pypi_host}/simple" }
-    let(:bundle) { fixture_file_content('python/pypi.crt') }
+  context "when connecting to a private package repository with self signed certificate" do
+    let(:index_url) { "https://pypi.test/simple" }
 
     before do
+      add_host('pypi.test', '127.0.0.1')
+      start_proxy_server
+
       runner.add_file('setup.py') do
         <<~RAW
 from setuptools import setup, find_packages
@@ -148,7 +150,7 @@ setup(
 
     it 'downloads the packages and trusts the certificate' do
       report = runner.scan(env: {
-        'ADDITIONAL_CA_CERT_BUNDLE' => bundle,
+        'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read,
         'PIP_INDEX_URL' => index_url
       })
 
diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb
index b95f1fa..5cc9128 100644
--- a/spec/integration/python/pipenv_spec.rb
+++ b/spec/integration/python/pipenv_spec.rb
@@ -202,18 +202,20 @@ RSpec.describe "pipenv" do
     end
   end
 
-  context "when connecting to a private package repository with self signed certificate", environment: 'offline' do
-    let(:index_url) { "https://#{private_pypi_host}/simple" }
-    let(:bundle) { fixture_file_content('python/pypi.crt') }
+  context "when connecting to a private package repository with self signed certificate" do
+    let(:index_url) { "https://pypi.test/simple" }
 
     before do
+      add_host('pypi.test', '127.0.0.1')
+      start_proxy_server
+
       runner.add_file('Pipfile', fixture_file_content('python/airgap-Pipfile.erb', index_url: index_url))
       runner.add_file('Pipfile.lock', fixture_file_content('python/airgap-Pipfile.lock.erb', index_url: index_url))
     end
 
     it 'downloads the packages and trusts the certificate' do
       report = runner.scan(env: {
-        'ADDITIONAL_CA_CERT_BUNDLE' => bundle,
+        'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read,
         'PIP_INDEX_URL' => index_url
       })
 
diff --git a/spec/support/integration_test_helper.rb b/spec/support/integration_test_helper.rb
index e482d6d..9d28871 100644
--- a/spec/support/integration_test_helper.rb
+++ b/spec/support/integration_test_helper.rb
@@ -1,16 +1,4 @@
 module IntegrationTestHelper
-  def private_npm_host
-    @private_npm_host ||= ENV.fetch('PRIVATE_NPM_HOST').tap do |host|
-      add_host(host, ENV.fetch('PRIVATE_NPM_IP'))
-    end
-  end
-
-  def private_pypi_host
-    @private_pypi_host ||= ENV.fetch('PRIVATE_PYPI_HOST').tap do |host|
-      add_host(host, ENV.fetch('PRIVATE_PYPI_IP'))
-    end
-  end
-
   def runner(*args)
     @runner ||= ProjectHelper.new(*args)
   end