Merge branch '212920-go-modules-tls' into 'master'

Add integration test to go modules TLS verification

See merge request gitlab-org/security-products/license-management!164

10 files changed, 148 insertions, 11 deletions

diff --git a/bin/setup b/bin/setup
index 7b7f9fb..b54a99f 100755
--- a/bin/setup
+++ b/bin/setup
@@ -6,3 +6,6 @@ cd "$(dirname "$0")/.."
 
 gem install bundler --conservative -v '~> 2.0' -q
 bundle install --quiet
+if [ ! -f /usr/sbin/haproxy ] && command -v apt-get; then
+  apt-get update -y && apt-get install -y --no-install-recommends haproxy
+fi
diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb
index 9868f1e..b24d24e 100644
--- a/lib/license/management/shell.rb
+++ b/lib/license/management/shell.rb
@@ -14,12 +14,12 @@ module License
 
       def execute(command, env: {})
         expanded_command = expand(command)
-        logger.debug(expanded_command)
-
-        stdout, stderr, status = Open3.capture3(env, expanded_command)
-
-        record(stdout, stderr, status)
-        [stdout, stderr, status]
+        collapsible_section(expanded_command) do
+          logger.debug(expanded_command)
+          stdout, stderr, status = Open3.capture3(env, expanded_command)
+          record(stdout, stderr, status)
+          [stdout, stderr, status]
+        end
       end
 
       def sh(command, env: {})
@@ -86,6 +86,14 @@ module License
           logger.error(stderr)
         end
       end
+
+      def collapsible_section(header)
+        id = header.downcase.gsub(/[[:space:]]/, '_').gsub(/[^0-9a-z ]/i, '_')
+        logger.debug("section_start:#{Time.now.to_i}:#{id}\r\e[0K#{header}")
+        yield
+      ensure
+        logger.debug("section_end:#{Time.now.to_i}:#{id}\r\e[0K")
+      end
     end
   end
 end
diff --git a/run.sh b/run.sh
index 8199f9e..04de6d3 100755
--- a/run.sh
+++ b/run.sh
@@ -52,10 +52,6 @@ function debug_env() {
 }
 
 function scan_project() {
-  gem install --local -f --silent "$LM_HOME/pkg/*.gem"
-  license_management ignored_groups add development
-  license_management ignored_groups add develop
-  license_management ignored_groups add test
   echo license_management report "$@"
   # shellcheck disable=SC2068
   license_management report $@
@@ -105,9 +101,16 @@ function prepare_project() {
     ${SETUP_CMD[@]}
     PREPARE="--no-prepare"
   fi
+  gem install --local -f --silent "$LM_HOME/pkg/*.gem"
+  license_management ignored_groups add development
+  license_management ignored_groups add develop
+  license_management ignored_groups add test
 }
 
+echo -e "section_start:$(date +%s):prepare_project\r\e[0KPrepare"
 prepare_project
+echo -e "section_end:$(date +%s):prepare_project\r\e[0K"
+
 [[ $CI_DEBUG_TRACE == 'true' ]] && debug_env
 
 scan_project "$PREPARE" \
diff --git a/spec/fixtures/go/1.14-ignore-tls/go.mod b/spec/fixtures/go/1.14-ignore-tls/go.mod
new file mode 100644
index 0000000..17847dd
--- /dev/null
+++ b/spec/fixtures/go/1.14-ignore-tls/go.mod
@@ -0,0 +1,5 @@
+module example.com/m
+
+go 1.14
+
+require github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0 // indirect
diff --git a/spec/fixtures/go/1.14-ignore-tls/go.sum b/spec/fixtures/go/1.14-ignore-tls/go.sum
new file mode 100644
index 0000000..b14d67a
--- /dev/null
+++ b/spec/fixtures/go/1.14-ignore-tls/go.sum
@@ -0,0 +1,9 @@
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0 h1:OggOMmdI0JLwg1FkOKH9S7fVHF0oEm8PX6S8kAdpOps=
+github.com/google/licenseclassifier v0.0.0-20200402202327-879cb1424de0/go.mod h1:qsqn2hxC+vURpyBRygGUuinTO42MFRLcsmQ/P8v94+M=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
diff --git a/spec/fixtures/haproxy.cfg b/spec/fixtures/haproxy.cfg
new file mode 100644
index 0000000..ef11e48
--- /dev/null
+++ b/spec/fixtures/haproxy.cfg
@@ -0,0 +1,28 @@
+global
+  chroot /var/lib/haproxy
+  ca-base ./tmp
+  crt-base ./tmp
+  tune.ssl.default-dh-param 4096
+
+defaults
+  log global
+  mode http
+  retries 1
+  timeout http-request    10s
+  timeout queue           1s
+  timeout connect         1s
+  timeout client          1s
+  timeout server          1s
+  timeout http-keep-alive 1s
+  timeout check           10s
+  maxconn 32
+
+frontend www-https
+  bind *:443 ssl crt wildcard.test.pem
+
+  acl goproxy-backend ssl_fc_sni goproxy.test
+  http-request replace-header Host .* proxy.golang.org if goproxy-backend
+  use_backend goproxy-backend if goproxy-backend
+
+backend goproxy-backend
+  server www1 proxy.golang.org:443 ssl verify none
diff --git a/spec/integration/go/modules_spec.rb b/spec/integration/go/modules_spec.rb
index 3be04af..fa2313c 100644
--- a/spec/integration/go/modules_spec.rb
+++ b/spec/integration/go/modules_spec.rb
@@ -89,4 +89,45 @@ RSpec.describe "modules" do
       expect(subject.licenses_for("golang.org/x/sys")).to match_array(['BSD-3-Clause'])
     end
   end
+
+  context "when scanning a project sourced from a TLS endpoint with a X.509 certificate signed by a private authority" do
+    subject { runner.scan(env: env) }
+
+    before do
+      add_host('goproxy.test', '127.0.0.1')
+      start_proxy_server
+      runner.mount(dir: fixture_file('go/1.14-ignore-tls'))
+    end
+
+    context "when the CA certificate is provided" do
+      let(:env) do
+        {
+          'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read,
+          'GOPROXY' => 'https://goproxy.test'
+        }
+      end
+
+      specify do
+        expect(subject).to match_schema
+        expect(subject.dependency_names).to match_array([
+          "github.com/davecgh/go-spew",
+          "github.com/google/go-cmp",
+          "github.com/google/licenseclassifier",
+          "github.com/pmezard/go-difflib",
+          "github.com/sergi/go-diff",
+          "github.com/stretchr/objx",
+          "github.com/stretchr/testify"
+        ])
+      end
+    end
+
+    context "when the CA certificate is NOT provided" do
+      let(:env) { { 'GOPROXY' => 'https://goproxy.test' } }
+
+      specify do
+        expect(subject).to match_schema
+        expect(subject.dependency_names).to be_empty
+      end
+    end
+  end
 end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index fab8418..c306997 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -2,11 +2,12 @@ require 'simplecov'
 
 require 'license/management'
 require 'json'
-require 'securerandom'
 require 'json-schema'
+require 'securerandom'
 require 'support/fixture_file_helper'
 require 'support/integration_test_helper'
 require 'support/matchers'
+require 'support/proxy_helper'
 require 'support/shared'
 
 RSpec.configure do |config|
diff --git a/spec/support/proxy_helper.rb b/spec/support/proxy_helper.rb
new file mode 100644
index 0000000..b79d0e3
--- /dev/null
+++ b/spec/support/proxy_helper.rb
@@ -0,0 +1,39 @@
+module ProxyHelper
+  def x509_certificate(host)
+    License::Management.root.join("tmp/#{host}.crt")
+  end
+
+  def generate_self_signed_certificate_for(host)
+    Dir.chdir License::Management.root.join('tmp') do
+      system([
+        "rm -f #{host}.*",
+        "openssl req -x509 -newkey rsa:4096 -keyout #{host}.key -out #{host}.crt -days 999 -nodes -subj \"/C=/ST=/L=/O=/OU=/CN=*.test\"",
+        "cat #{host}.* > #{host}.pem"
+      ].join("&&"))
+    end
+  end
+
+  def start_proxy_server
+    @proxy_server_pid ||=
+      begin
+        generate_self_signed_certificate_for('wildcard.test')
+        spawn("/usr/sbin/haproxy -f #{fixture_file('haproxy.cfg')}")
+      end
+  end
+
+  def stop_proxy_server
+    return if !defined?(@proxy_server_pid) || @proxy_server_pid.nil?
+
+    Process.kill("TERM", @proxy_server_pid)
+    Process.wait(@proxy_server_pid)
+    @proxy_server_pid = nil
+  end
+end
+
+RSpec.configure do |config|
+  config.include(ProxyHelper, type: :integration)
+
+  config.after(:example, type: :integration) do
+    stop_proxy_server
+  end
+end
diff --git a/tmp/.keep b/tmp/.keep
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tmp/.keep