Merge branch '33727-airgap' into 'master'v3.1.0

Scan maven projects with dependencies from a custom repository

See merge request gitlab-org/security-products/license-management!120

15 files changed, 175 insertions, 46 deletions

diff --git a/CHANGELOG.md b/CHANGELOG.md
index d1a77a4..4538573 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # GitLab License management changelog
 
+## v3.1.0
+
+- Forward custom `MAVEN_CLI_OPTS` to `LicenseFinder` so that it can use it in the license scan task. (!120)
+
 ## v3.0.0
 
 - Use asdf version manager to install custom tools (!98)
diff --git a/Dockerfile b/Dockerfile
index ad0598b..c8e8610 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -11,6 +11,12 @@ RUN apt-get update -q \
   && apt-get install -y --no-install-recommends ruby \
   && gem build *.gemspec
 
+# Install org.codehaus.mojo:license-maven-plugin:download-licenses to $HOME/.m2/repository
+FROM debian:stable AS license-maven-plugin-builder
+RUN apt-get update -q \
+  && apt-get install -y --no-install-recommends maven \
+  && mvn license:help
+
 FROM debian:stable-slim as tools-builder
 ENV ASDF_DATA_DIR="/opt/asdf"
 ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:${PATH}"
@@ -23,6 +29,7 @@ RUN bash /root/install.sh
 
 FROM tools-builder
 ENV LM_HOME=/opt/license-management
+COPY --from=license-maven-plugin-builder /root/.m2/repository/* /root/.m2/repository/
 COPY --from=gem-builder /opt/license-management/*.gem $LM_HOME/pkg/
 COPY run.sh /
 ENTRYPOINT ["/run.sh"]
diff --git a/Gemfile.lock b/Gemfile.lock
index c629f11..7dfdc77 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    license-management (3.0.0)
+    license-management (3.1.0)
       license_finder (~> 6.0.0)
 
 GEM
@@ -40,7 +40,7 @@ GEM
       rspec-mocks (~> 3.9.0)
     rspec-core (3.9.1)
       rspec-support (~> 3.9.1)
-    rspec-expectations (3.9.0)
+    rspec-expectations (3.9.1)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.9.0)
     rspec-mocks (3.9.1)
@@ -66,7 +66,7 @@ GEM
     rubocop-rspec (1.38.1)
       rubocop (>= 0.68.1)
     ruby-progressbar (1.10.1)
-    rubyzip (2.2.0)
+    rubyzip (2.3.0)
     thor (1.0.1)
     toml (0.2.0)
       parslet (~> 1.8.0)
@@ -85,4 +85,4 @@ DEPENDENCIES
   rspec_junit_formatter (~> 0.4)
 
 BUNDLED WITH
-   2.1.2
+   2.1.4
diff --git a/config/.m2/settings.xml b/config/.m2/settings.xml
new file mode 100644
index 0000000..c36a363
--- /dev/null
+++ b/config/.m2/settings.xml
@@ -0,0 +1,32 @@
+<settings xmlns="http://maven.apache.org/SETTINGS/1.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/SETTINGS/1.0.0 https://maven.apache.org/xsd/settings-1.0.0.xsd">
+  <localRepository>${user.home}/.m2/repository</localRepository>
+  <interactiveMode>false</interactiveMode>
+  <offline>false</offline>
+  <profiles>
+    <profile>
+      <id>gitlab-maven</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <repositories>
+        <repository>
+          <id>gitlab-maven</id>
+          <url>https://gitlab.com/api/v4/projects/${env.CI_PROJECT_ID}/packages/maven</url>
+        </repository>
+      </repositories>
+    </profile>
+  </profiles>
+  <servers>
+    <server>
+      <id>gitlab-maven</id>
+      <configuration>
+        <httpHeaders>
+          <property>
+            <name>Job-Token</name>
+            <value>${env.CI_JOB_TOKEN}</value>
+          </property>
+        </httpHeaders>
+      </configuration>
+    </server>
+  </servers>
+</settings>
diff --git a/config/install.sh b/config/install.sh
index 873cb14..0a0ddf2 100644
--- a/config/install.sh
+++ b/config/install.sh
@@ -1,5 +1,5 @@
 #!/bin/bash
-set -euo pipefail
+set -euxo pipefail
 
 export DEBIAN_FRONTEND=noninteractive
 
@@ -85,6 +85,9 @@ git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR"
 cd "$ASDF_DATA_DIR"
 git checkout "$(git describe --abbrev=0 --tags)"
 
+# shellcheck source=/dev/null
+. "$ASDF_DATA_DIR"/asdf.sh
+
 while IFS= read -r line; do
   tool=$(echo "$line" | cut -d' ' -f1)
   asdf plugin-add "$tool"
@@ -93,6 +96,7 @@ bash "$ASDF_DATA_DIR/plugins/nodejs/bin/import-release-team-keyring"
 asdf install
 asdf reshim
 asdf current
+
 rm -fr /tmp
 mkdir -p /tmp
 chmod 777 /tmp
diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb
index c431b6b..9788abb 100644
--- a/lib/license/finder/ext.rb
+++ b/lib/license/finder/ext.rb
@@ -1,38 +1,8 @@
 # frozen_string_literal: true
 
+require 'license/finder/ext/license'
+require 'license/finder/ext/nuget'
+require 'license/finder/ext/shared_helpers'
+
 # Apply patch to the JsonReport found in the `license_finder` gem.
 LicenseFinder::JsonReport.prepend(License::Management::Report)
-
-module LicenseFinder
-  class Nuget
-    def prepare_command
-      "mono /usr/local/bin/nuget.exe restore -Verbosity detailed"
-    end
-
-    def license_urls(dependency)
-      filename = "#{dependency.name}.#{dependency.version}.nupkg"
-      files = Dir["**/#{filename}"] +
-        Dir.glob(File.join(Dir.home, '.nuget', 'packages', '**', '**', filename.downcase))
-
-      return if files.empty?
-
-      Zip::File.open(files.first) do |zipfile|
-        Nuget.nuspec_license_urls(zipfile.read(dependency.name + '.nuspec'))
-      end
-    end
-  end
-
-  module SharedHelpers
-    class Cmd
-      def self.run(command)
-        stdout, stderr, status = Open3.capture3(command)
-        ::LicenseFinder::Core.default_logger.info(self.class, stdout) if ENV['CI_DEBUG_TRACE'] == 'true'
-        [stdout, stderr, status]
-      end
-    end
-  end
-
-  class License
-    attr_reader :short_name, :pretty_name, :other_names
-  end
-end
diff --git a/lib/license/finder/ext/license.rb b/lib/license/finder/ext/license.rb
new file mode 100644
index 0000000..cca3bd7
--- /dev/null
+++ b/lib/license/finder/ext/license.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module LicenseFinder
+  class License
+    attr_reader :short_name, :pretty_name, :other_names
+  end
+end
diff --git a/lib/license/finder/ext/nuget.rb b/lib/license/finder/ext/nuget.rb
new file mode 100644
index 0000000..fc06e21
--- /dev/null
+++ b/lib/license/finder/ext/nuget.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module LicenseFinder
+  class Nuget
+    def prepare_command
+      "mono /usr/local/bin/nuget.exe restore -Verbosity detailed"
+    end
+
+    def license_urls(dependency)
+      filename = "#{dependency.name}.#{dependency.version}.nupkg"
+      files = Dir["**/#{filename}"] +
+        Dir.glob(File.join(Dir.home, '.nuget', 'packages', '**', '**', filename.downcase))
+
+      return if files.empty?
+
+      Zip::File.open(files.first) do |zipfile|
+        Nuget.nuspec_license_urls(zipfile.read(dependency.name + '.nuspec'))
+      end
+    end
+  end
+end
diff --git a/lib/license/finder/ext/shared_helpers.rb b/lib/license/finder/ext/shared_helpers.rb
new file mode 100644
index 0000000..4e61261
--- /dev/null
+++ b/lib/license/finder/ext/shared_helpers.rb
@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module LicenseFinder
+  module SharedHelpers
+    class Cmd
+      def self.run(command)
+        stdout, stderr, status = Open3.capture3(command)
+        debug([command, stdout].join('\n'))
+        [stdout, stderr, status]
+      end
+
+      def self.debug(message)
+        return unless ENV['CI_DEBUG_TRACE'] == 'true'
+
+        ::LicenseFinder::Core.default_logger.info(self.class, message)
+      end
+    end
+  end
+end
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
index 90b1ffa..ca107df 100644
--- a/lib/license/management/version.rb
+++ b/lib/license/management/version.rb
@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '3.0.0'
+    VERSION = '3.1.0'
   end
 end
diff --git a/run.sh b/run.sh
index c4c98e7..e18ae43 100755
--- a/run.sh
+++ b/run.sh
@@ -6,12 +6,15 @@ set -e
 BUNDLE_JOBS="$(nproc)"
 export BUNDLE_JOBS
 export BUNDLE_WITHOUT="development:test"
-export CI_DEBUG_TRACE=${CI_DEBUG_TRACE:-}
+export CI_DEBUG_TRACE=${CI_DEBUG_TRACE:='false'}
 export DOTNET_CLI_TELEMETRY_OPTOUT=1
 export HISTFILESIZE=0
 export HISTSIZE=0
+export LICENSE_FINDER_CLI_OPTS=${LICENSE_FINDER_CLI_OPTS:='--no-debug'}
 export LM_REPORT_FILE=${LM_REPORT_FILE:-'gl-license-management-report.json'}
+export MAVEN_CLI_OPTS="${MAVEN_CLI_OPTS:='-DskipTests'}"
 export PREPARE="--prepare-no-fail"
+export RECURSIVE='--no-recursive'
 export RUBY_GC_HEAP_INIT_SLOTS=800000
 export RUBY_GC_MALLOC_LIMIT=79000000
 export RUBY_HEAP_FREE_MIN=100000
@@ -66,14 +69,12 @@ function prepare_golang() {
 }
 
 function prepare_java() {
-  [[ -f pom.xml ]] && mvn install "${MAVEN_CLI_OPTS:--DskipTests}"
   [[ -f build.gradle ]] && [[ ! -f gradlew ]] && \
     gradle build ${GRADLE_CLI_OPTS:+-x test}
 }
 
 function prepare_dotnet() {
-  [[ $(ls ./*.sln 2> /dev/null) ]] && \
-    LICENSE_FINDER_CLI_OPTS="--recursive=true ${LICENSE_FINDER_CLI_OPTS:-}"
+  [[ $(ls ./*.sln 2> /dev/null) ]] && RECURSIVE="--recursive"
 }
 
 function prepare_project() {
@@ -104,8 +105,11 @@ switch_to java "adopt-openjdk-${LM_JAVA_VERSION:-8}"
 
 prepare_project
 [[ $CI_DEBUG_TRACE == 'true' ]] && debug_env
+
 scan_project "$PREPARE" \
   --format=json \
   --save="${LM_REPORT_FILE}" \
   --python-version="${python_version}" \
-  ${LICENSE_FINDER_CLI_OPTS:+--no-debug $LICENSE_FINDER_CLI_OPTS}
+  "$RECURSIVE" \
+  --maven-options="$MAVEN_CLI_OPTS" \
+  "$LICENSE_FINDER_CLI_OPTS"
diff --git a/spec/fixtures/custom-maven-settings.xml b/spec/fixtures/custom-maven-settings.xml
new file mode 100644
index 0000000..4fa5d16
--- /dev/null
+++ b/spec/fixtures/custom-maven-settings.xml
@@ -0,0 +1,16 @@
+<settings>
+  <profiles>
+    <profile>
+      <id>custom</id>
+      <activation>
+        <activeByDefault>true</activeByDefault>
+      </activation>
+      <repositories>
+        <repository>
+          <id>gitlab-maven</id>
+          <url>https://gitlab.com/api/v4/projects/17523603/packages/maven</url>
+        </repository>
+      </repositories>
+    </profile>
+  </profiles>
+</settings>
diff --git a/spec/fixtures/pom-public-gitlab-repository.xml b/spec/fixtures/pom-public-gitlab-repository.xml
new file mode 100644
index 0000000..4e57c79
--- /dev/null
+++ b/spec/fixtures/pom-public-gitlab-repository.xml
@@ -0,0 +1,16 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+  <modelVersion>4.0.0</modelVersion>
+  <groupId>com.gitlab.secure</groupId>
+  <artifactId>license-scanning</artifactId>
+  <packaging>jar</packaging>
+  <version>1.0-SNAPSHOT</version>
+  <name>example</name>
+  <url>http://maven.apache.org</url>
+  <dependencies>
+    <dependency>
+      <groupId>com.gitlab.xlgmokha</groupId>
+      <artifactId>mvn-spike</artifactId>
+      <version>1.2-SNAPSHOT</version>
+    </dependency>
+  </dependencies>
+</project>
diff --git a/spec/integration/java/maven_spec.rb b/spec/integration/java/maven_spec.rb
index 6e1752f..8d1ad20 100644
--- a/spec/integration/java/maven_spec.rb
+++ b/spec/integration/java/maven_spec.rb
@@ -2,4 +2,29 @@ require 'spec_helper'
 
 RSpec.describe "maven" do
   include_examples "each report version", "java", "maven"
+
+  describe "When the maven dependencies come from a custom public maven repository" do
+    it 'is able to detect some of the licenses' do
+      runner.add_file('pom.xml', fixture_file_content('pom-public-gitlab-repository.xml'))
+
+      report = runner.scan(env: { 'CI_PROJECT_ID' => '17523603' })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).to match_array([{ name: 'mvn-spike', url: '', description: '', paths: ['.'], licenses: ['MIT'] }])
+    end
+
+    it 'downloads packages from by using a custom `settings.xml`' do
+      runner.add_file('pom.xml', fixture_file_content('pom-public-gitlab-repository.xml'))
+      runner.add_file('my_settings.xml', fixture_file_content('custom-maven-settings.xml'))
+
+      report = runner.scan(env: {
+        'CI_DEBUG_TRACE' => 'true',
+        'CI_PROJECT_ID' => 'invalid',
+        'MAVEN_CLI_OPTS' => "--settings my_settings.xml"
+      })
+
+      expect(report).to match_schema(version: '2.0')
+      expect(report[:dependencies]).to match_array([{ name: 'mvn-spike', url: '', description: '', paths: ['.'], licenses: ['MIT'] }])
+    end
+  end
 end
diff --git a/spec/support/fixture_file_helper.rb b/spec/support/fixture_file_helper.rb
index c98b98a..fe11acd 100644
--- a/spec/support/fixture_file_helper.rb
+++ b/spec/support/fixture_file_helper.rb
@@ -1,5 +1,9 @@
 module FixtureFileHelper
   def fixture_file_content(path)
-    IO.read(License::Management.root.join("spec/fixtures/#{path}"))
+    IO.read(fixture_file(path))
+  end
+
+  def fixture_file(path)
+    License::Management.root.join("spec/fixtures/#{path}")
   end
 end