diff options
| -rw-r--r-- | CHANGELOG.md | 4 | ||||
| -rw-r--r-- | Gemfile.lock | 6 | ||||
| -rwxr-xr-x | bin/docker-test | 2 | ||||
| -rwxr-xr-x | bin/test | 2 | ||||
| -rw-r--r-- | config/.bashrc | 13 | ||||
| -rw-r--r-- | config/.tool-versions | 2 | ||||
| -rw-r--r-- | lib/license/finder/ext.rb | 2 | ||||
| -rw-r--r-- | lib/license/finder/ext/gradle.rb | 50 | ||||
| -rw-r--r-- | lib/license/finder/ext/package_manager.rb | 14 | ||||
| -rw-r--r-- | lib/license/finder/ext/pip.rb | 14 | ||||
| -rw-r--r-- | lib/license/finder/ext/pipenv.rb | 8 | ||||
| -rw-r--r-- | lib/license/management.rb | 6 | ||||
| -rw-r--r-- | lib/license/management/shell.rb | 1 | ||||
| -rw-r--r-- | lib/license/management/version.rb | 2 | ||||
| -rwxr-xr-x | run.sh | 25 | ||||
| -rw-r--r-- | spec/fixtures/java/11/build.gradle | 13 | ||||
| -rw-r--r-- | spec/fixtures/java/8/build.gradle | 9 | ||||
| -rw-r--r-- | spec/fixtures/java/build.gradle.kts | 12 | ||||
| -rw-r--r-- | spec/integration/java/gradle_spec.rb | 77 |
19 files changed, 203 insertions, 59 deletions
diff --git a/CHANGELOG.md b/CHANGELOG.md index 54a2fde..5def309 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # GitLab License management changelog +## v3.7.2 + +- Forward custom `GRADLE_CLI_OPTS` to `gradle downloadLicenses` and skip additional install step. (!121) + ## v3.7.1 - Export `PIP_CERT` when invoking `pip` when a custom root certificate is specified. (!133) diff --git a/Gemfile.lock b/Gemfile.lock index 3b282f0..ce54189 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - license-management (3.7.1) + license-management (3.7.2) license_finder (~> 6.0.0) spandx (~> 0.1) @@ -72,18 +72,20 @@ GEM rubocop (>= 0.68.1) ruby-progressbar (1.10.1) rubyzip (2.3.0) - spandx (0.11.0) + spandx (0.12.0) addressable (~> 2.7) bundler (>= 1.16, < 3.0.0) net-hippie (~> 0.3) nokogiri (~> 1.10) thor + zeitwerk (~> 2.3) thor (1.0.1) toml (0.2.0) parslet (~> 1.8.0) unicode-display_width (1.6.1) with_env (1.1.0) xml-simple (1.1.5) + zeitwerk (2.3.0) PLATFORMS ruby diff --git a/bin/docker-test b/bin/docker-test index 5a4436d..6fac5d7 100755 --- a/bin/docker-test +++ b/bin/docker-test @@ -5,7 +5,6 @@ set -eu cd "$(dirname "$0")/.." export CI_PROJECT_DIR="$1" -echo "Scanning $CI_PROJECT_DIR" if [ -n "${LM_HOME:-}" ]; then mkdir -p pkg/ && gem build --silent -o pkg/license-management-test.gem ./*.gemspec @@ -14,6 +13,7 @@ else export IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest} docker run --rm \ + --env GRADLE_CLI_OPTS \ --env LICENSE_FINDER_CLI_OPTS \ --env LM_JAVA_VERSION \ --env LM_PYTHON_VERSION \ @@ -5,4 +5,4 @@ set -e cd "$(dirname "$0")/.." ./bin/setup -bundle exec rspec "$@" --format=documentation +bundle exec rspec "$@" --format=progress diff --git a/config/.bashrc b/config/.bashrc index 288d919..9c463dd 100644 --- a/config/.bashrc +++ b/config/.bashrc @@ -11,12 +11,25 @@ function inflate() { fi } +update_java_home() { + local java_path + java_path="$(asdf which java)" + if [[ -n "${java_path}" ]]; then + export JAVA_HOME + JAVA_HOME="$(dirname "$(dirname "$(realpath "${java_path}")")")" + fi +} + function switch_to() { local tool=$1 local major_version=$2 local version version="$(grep "$tool" "$HOME/.tool-versions"| tr ' ' '\n' | grep "^$major_version")" asdf shell "$tool" "$version" + + if [[ "$tool" = "java" ]]; then + update_java_home + fi } function major_version_from() { diff --git a/config/.tool-versions b/config/.tool-versions index 92ddb32..9803064 100644 --- a/config/.tool-versions +++ b/config/.tool-versions @@ -1,7 +1,7 @@ elixir system golang system gradle 6.3 -java adopt-openjdk-8u242-b08 adopt-openjdk-11.0.6+10 +java adopt-openjdk-8u242-b08 adopt-openjdk-11.0.7+10 maven 3.6.3 nodejs 10.19.0 php 7.4.4 diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb index 24afd37..70620be 100644 --- a/lib/license/finder/ext.rb +++ b/lib/license/finder/ext.rb @@ -1,9 +1,11 @@ # frozen_string_literal: true require 'license/finder/ext/go_modules' +require 'license/finder/ext/gradle' require 'license/finder/ext/license' require 'license/finder/ext/maven' require 'license/finder/ext/nuget' +require 'license/finder/ext/package_manager' require 'license/finder/ext/pip' require 'license/finder/ext/pipenv' require 'license/finder/ext/shared_helpers' diff --git a/lib/license/finder/ext/gradle.rb b/lib/license/finder/ext/gradle.rb new file mode 100644 index 0000000..2c3ce01 --- /dev/null +++ b/lib/license/finder/ext/gradle.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +module LicenseFinder + class Gradle + def current_packages + return [] unless download_licenses + + Pathname + .glob(project_path.join('**', 'dependency-license.xml')) + .map(&:read) + .flat_map { |xml_file| parse_from(xml_file) }.uniq + end + + def package_management_command + wrapper? ? './gradlew' : 'gradle' + end + + private + + def download_licenses + _stdout, _stderr, status = Dir.chdir(project_path) do + shell.execute([ + @command, + ENV.fetch('GRADLE_CLI_OPTS', '--exclude-task=test'), + 'downloadLicenses' + ], env: { 'TERM' => 'noop' }) + end + + status.success? + end + + def wrapper? + File.exist?(File.join(project_path, 'gradlew')) + end + + def xml_parsing_options + @xml_parsing_options ||= { 'GroupTags' => { 'dependencies' => 'dependency' } } + end + + def parse_from(xml_file) + XmlSimple + .xml_in(xml_file, xml_parsing_options) + .fetch('dependency', []).map { |hash| map_from(hash) } + end + + def map_from(hash) + GradlePackage.new(hash, include_groups: @include_groups) + end + end +end diff --git a/lib/license/finder/ext/package_manager.rb b/lib/license/finder/ext/package_manager.rb new file mode 100644 index 0000000..ff5466e --- /dev/null +++ b/lib/license/finder/ext/package_manager.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module LicenseFinder + class PackageManager + def current_packages_with_relations + current_packages + rescue StandardError => e + ::License::Management.logger.error(e) + raise e unless @prepare_no_fail + + [] + end + end +end diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb index 084816c..51e2039 100644 --- a/lib/license/finder/ext/pip.rb +++ b/lib/license/finder/ext/pip.rb @@ -55,19 +55,11 @@ module LicenseFinder Dir.chdir(project_path) { yield } end - def pypi - @pypi ||= Spandx::Python::PyPI.new(sources: [ - Spandx::Python::Source.new({ - 'name' => 'pypi', - 'url' => python.pip_index_url, - 'verify_ssl' => true - }) - ]) - end - def legacy_results + sources = [Spandx::Python::Source.new({ 'name' => 'pypi', 'url' => python.pip_index_url, 'verify_ssl' => true })] + pypi = Spandx::Python::PyPI.new pip_output.map do |name, version, _children, _location| - spec = pypi.definition_for(name, version) + spec = pypi.definition_for(name, version, sources: sources) Package.new( name, version, diff --git a/lib/license/finder/ext/pipenv.rb b/lib/license/finder/ext/pipenv.rb index 056b81d..ebe8cad 100644 --- a/lib/license/finder/ext/pipenv.rb +++ b/lib/license/finder/ext/pipenv.rb @@ -38,11 +38,15 @@ module LicenseFinder end def build_package_for(name, version) - PipPackage.new(name, version, pypi.definition_for(name, version)) + PipPackage.new(name, version, pypi.definition_for(name, version, sources: sources)) + end + + def sources + @sources ||= ::Spandx::Python::Source.sources_from(lockfile_hash) end def pypi - @pypi ||= ::Spandx::Python::PyPI.new(sources: ::Spandx::Python::Source.sources_from(lockfile_hash)) + @pypi ||= ::Spandx::Python::Pypi.new end def lockfile_hash diff --git a/lib/license/management.rb b/lib/license/management.rb index e156d42..4be002d 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -24,7 +24,11 @@ module License end def self.logger - @logger ||= Logger.new(STDOUT, level: ENV.fetch('LOG_LEVEL', Logger::WARN)) + @logger ||= Logger.new(STDOUT, level: ENV.fetch('LOG_LEVEL', Logger::WARN)).tap do |x| + x.formatter = proc do |severity, _datetime, _progname, message| + "#{severity} -- : #{message}\n" + end + end end def self.shell diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb index 8850e60..6720460 100644 --- a/lib/license/management/shell.rb +++ b/lib/license/management/shell.rb @@ -40,6 +40,7 @@ module License return unless present?(certificate) custom_certificate_path.write(certificate) + execute("openssl x509 -in #{custom_certificate_path} -text -noout") execute('update-ca-certificates -v') end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index fe617f9..46ec201 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.7.1' + VERSION = '3.7.2' end end @@ -63,43 +63,38 @@ function prepare_javascript() { fi } -function prepare_java() { - [[ -f build.gradle ]] && [[ ! -f gradlew ]] && \ - gradle build ${GRADLE_CLI_OPTS:+-x test} -} - function prepare_dotnet() { [[ $(ls ./*.sln 2> /dev/null) ]] && RECURSIVE="--recursive" } +function prepare_tools() { + if ! asdf current 2> >(grep -q 'is not installed'); then + echo "Installing missing tools…" + asdf install > /dev/null 2>&1 + fi +} + function prepare_project() { if [[ -z ${SETUP_CMD:-} ]]; then - asdf install 1> /dev/null - + prepare_tools || true prepare_javascript || true - prepare_java || true prepare_dotnet || true else - echo "Running '${SETUP_CMD}' to install project dependencies..." + echo "Running '${SETUP_CMD}' to install project dependencies…" # shellcheck disable=SC2068 ${SETUP_CMD[@]} PREPARE="--no-prepare" fi } -python_version=$(major_version_from "${LM_PYTHON_VERSION:-3}") -switch_to python "$python_version" +switch_to python "$(major_version_from "${LM_PYTHON_VERSION:-3}")" switch_to java "adopt-openjdk-${LM_JAVA_VERSION:-8}" -# shellcheck source=/dev/null -. "$ASDF_DATA_DIR/plugins/java/set-java-home.sh" - prepare_project [[ $CI_DEBUG_TRACE == 'true' ]] && debug_env scan_project "$PREPARE" \ --format=json \ --save="${LM_REPORT_FILE}" \ - --python-version="${python_version}" \ "$RECURSIVE" \ "$LICENSE_FINDER_CLI_OPTS" diff --git a/spec/fixtures/java/11/build.gradle b/spec/fixtures/java/11/build.gradle new file mode 100644 index 0000000..fa128ea --- /dev/null +++ b/spec/fixtures/java/11/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'java-library' +} + +repositories { + jcenter() +} + +dependencies { + runtime "org.postgresql:postgresql:42.1.4" + implementation 'com.google.guava:guava:28.2-jre' + testImplementation 'junit:junit:4.12' +} diff --git a/spec/fixtures/java/8/build.gradle b/spec/fixtures/java/8/build.gradle new file mode 100644 index 0000000..b7cffcd --- /dev/null +++ b/spec/fixtures/java/8/build.gradle @@ -0,0 +1,9 @@ +apply plugin: 'groovy' + +repositories { + mavenCentral() +} + +dependencies { + compile 'org.postgresql:postgresql:42.1.4' +} diff --git a/spec/fixtures/java/build.gradle.kts b/spec/fixtures/java/build.gradle.kts index 494fc8b..2ca8866 100644 --- a/spec/fixtures/java/build.gradle.kts +++ b/spec/fixtures/java/build.gradle.kts @@ -1,11 +1,13 @@ plugins { - `java-library` + `java-library` } + repositories { - jcenter() + jcenter() } + dependencies { - api("org.apache.commons:commons-math3:3.6.1") - implementation("com.google.guava:guava:28.1-jre") - testImplementation("junit:junit:4.12") + runtime("org.postgresql:postgresql:42.1.4") + implementation("com.google.guava:guava:28.1-jre") + testImplementation("junit:junit:4.12") } diff --git a/spec/integration/java/gradle_spec.rb b/spec/integration/java/gradle_spec.rb index 1b566bb..b81e69f 100644 --- a/spec/integration/java/gradle_spec.rb +++ b/spec/integration/java/gradle_spec.rb @@ -49,28 +49,67 @@ plugins { end end + context "when scanning a gradle project with a custom option to generate a profiler report" do + let(:report) { runner.scan(env: { 'GRADLE_CLI_OPTS' => '--profile' }) } + + before do + runner.add_file('build.gradle', fixture_file_content("java/11/build.gradle")) + end + + specify { expect(report).to match_schema(version: '2.0') } + specify { expect { report }.to change { Dir.glob("#{runner.project_path}/build/reports/profile/profile-*.html").count }.from(0).to(1) } + specify { expect(report.dependency_names).to match_array(['postgresql']) } + specify { expect(report.licenses_for('postgresql')).to match_array(['BSD-2-Clause']) } + end + + context 'when using Java 8 with version 1.* of gradle' do + before do + runner.add_file('.tool-versions', "gradle 1.9") + runner.add_file('build.gradle', fixture_file_content("java/8/build.gradle")) + end + + it 'returns an empty report because the plugin we use does not work in this version of the gradle API' do + report = runner.scan(env: { 'LM_JAVA_VERSION' => '8' }) + + expect(report).to match_schema(version: '2.0') + expect(report[:dependencies]).to be_empty + expect(report[:licenses]).to be_empty + end + end + + ['4.9', '5.6', '6.3'].each do |gradle_version| + context "when using Java v11 with a kotlin project using gradle v#{gradle_version}" do + let(:report) { runner.scan(env: { 'LM_JAVA_VERSION' => '11', 'GRADLE_CLI_OPTS' => '-b build.gradle.kts' }) } + + before do + runner.add_file('.tool-versions', "gradle #{gradle_version}") + runner.add_file('build.gradle.kts', fixture_file_content("java/build.gradle.kts")) + runner.add_file('settings.gradle.kts', 'rootProject.name = "example"') + end + + specify { expect(report).to match_schema(version: '2.0') } + specify { expect(report.dependency_names).to match_array(['postgresql']) } + specify { expect(report.licenses_for('postgresql')).to match_array(['BSD-2-Clause']) } + end + end + [ - '6.2', - '5.6', - '4.9', - '3.5', - '2.9', - '1.9' - ].each do |gradle_version| - %w[8 11].each do |java_version| - context "when scanning a gradle (v#{gradle_version}) project that uses a kotlin build script" do - let(:build_file_content) { fixture_file_content("java/build.gradle.kts") } - - it 'scans a gradle project' do - runner.add_file('build.gradle.kts', build_file_content) - runner.add_file('settings.gradle.kts', 'rootProject.name = "example"') - runner.add_file('.tool-versions', "gradle #{gradle_version}") + { java: '8', gradle: ['2.9', '3.5'] }, + { java: '11', gradle: ['4.9', '5.6', '6.3'] } + ].each do |item| + item[:gradle].each do |gradle_version| + context "when using Java v#{item[:java]} with a gradle v#{gradle_version} on a groovy project" do + let(:report) { runner.scan(env: { 'LM_JAVA_VERSION' => item[:java] }) } - report = runner.scan(env: { 'LM_JAVA_VERSION' => java_version }) - expect(report).to match_schema(version: '2.0') - expect(report[:licenses]).to be_empty - expect(report[:dependencies]).to be_empty + before do + runner.add_file('.tool-versions', "gradle #{gradle_version}") + runner.add_file('build.gradle', fixture_file_content("java/#{item[:java]}/build.gradle")) + runner.add_file('settings.gradle', 'rootProject.name = "example"') end + + specify { expect(report).to match_schema(version: '2.0') } + specify { expect(report.dependency_names).to match_array(['postgresql']) } + specify { expect(report.licenses_for('postgresql')).to match_array(['BSD-2-Clause']) } end end end |