diff options
25 files changed, 423 insertions, 8 deletions
diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..6b8710a --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +.git diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a9a5aec --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +tmp diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index d54da14..464757f 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -35,13 +35,15 @@ container_scanning: image: docker:stable stage: test variables: + FEATURE_RUBY_REPORT: 'false' LM_PYTHON_VERSION: 2 + LM_REPORT_VERSION: 1 script: - docker info - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - docker pull $TMP_IMAGE - mkdir results - - docker run --env LM_PYTHON_VERSION --volume `pwd`/results:/results $TMP_IMAGE test $QA_PROJECT ${QA_RESULTS:-$QA_PROJECT} $QA_REF + - ./bin/test artifacts: paths: - results/ @@ -63,7 +65,7 @@ QA:python3-pip: extends: .QA variables: LM_PYTHON_VERSION: 3 - QA_RESULTS: python3-pip + QA_RESULTS: python3-pip-v1 QA_PROJECT: python-pip QA_REF: 48e250a1 diff --git a/CHANGELOG.md b/CHANGELOG.md index 57c9bd2..73eac53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # GitLab License management changelog +## v1.6.0 + +- Convert HTML to JSON transformation to generating a JSON report directly. + + ## v1.5.0 - Reverts 1.4.0 @@ -22,6 +22,9 @@ RUN apt-get update && \ libpq-dev libmysqlclient-dev realpath python3-dev python3-pip && \ rm -rf /var/lib/apt/lists/* +COPY . /opt/license-management/ +RUN bash -lc "cd /opt/license-management && gem build *.gemspec && gem install *.gem" + # Don't load RVM automatically, it doesn't work with GitLab-CI RUN mv /etc/profile.d/rvm.sh /rvm.sh @@ -43,6 +46,7 @@ RUN pip3 install --disable-pip-version-check setuptools==$SETUPTOOLS_VERSION && # Version of Python, defaults to Python 2.7 ARG LM_PYTHON_VERSION ENV LM_PYTHON_VERSION ${LM_PYTHON_VERSION:-2.7} +ENV LM_REPORT_VERSION ${LM_REPORT_VERSION:-1} COPY test /test COPY run.sh html2json.js / @@ -40,7 +40,7 @@ docker run --rm --volume "/path/to/my/project":/code license-management analyze You can run integration tests on the image like this: ```sh -docker run --rm license-management /test/test.sh project_name git_ref +./bin/test_all ``` where: diff --git a/bin/test b/bin/test new file mode 100755 index 0000000..c1cd989 --- /dev/null +++ b/bin/test @@ -0,0 +1,17 @@ +#!/bin/sh + +set -e + +export FEATURE_RUBY_REPORT=${FEATURE_RUBY_REPORT:-true} +export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-2} +export LM_REPORT_VERSION=${LM_REPORT_VERSION:-1} +export QA_RESULTS=${QA_RESULTS:-$QA_PROJECT-v$LM_REPORT_VERSION} +export RESULTS_DIR=${RESULTS_DIR:-`pwd`/results} + +docker run \ + --rm \ + --env FEATURE_RUBY_REPORT \ + --env LM_PYTHON_VERSION \ + --env LM_REPORT_VERSION \ + --volume $RESULTS_DIR:/results \ + $TMP_IMAGE test $QA_PROJECT ${QA_RESULTS} $QA_REF diff --git a/bin/test_all b/bin/test_all new file mode 100755 index 0000000..1aa19b4 --- /dev/null +++ b/bin/test_all @@ -0,0 +1,21 @@ +#!/bin/sh + +set -e + +export RESULTS_DIR=`pwd`/tmp +export TMP_IMAGE=$(basename $PWD):latest + +docker pull licensefinder/license_finder:5.6.2 +docker build . -t $TMP_IMAGE + +REPORT_VERSIONS=(1) +for version in "${REPORT_VERSIONS[@]}" +do + export LM_REPORT_VERSION=$version + echo "Report Version $LM_REPORT_VERSION" + QA_PROJECT=go-modules QA_RESULTS="go-modules-v$version" QA_REF=master ./bin/test + QA_PROJECT=java-maven QA_RESULTS="java-maven-v$version" QA_REF=831c7a04 ./bin/test + QA_PROJECT=python-pip QA_RESULTS="python-pip-v$version" QA_REF=04dce91b LM_PYTHON_VERSION=2 ./bin/test + QA_PROJECT=python-pip QA_RESULTS="python3-pip-v$version" QA_REF=48e250a1 LM_PYTHON_VERSION=3 ./bin/test + QA_PROJECT=ruby-bundler QA_RESULTS="ruby-bundler-v$version" QA_REF=6b858821 ./bin/test +done diff --git a/exe/license_management b/exe/license_management new file mode 100755 index 0000000..33e110e --- /dev/null +++ b/exe/license_management @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'license/management' + +LicenseFinder::CLI::Main.start(ARGV) diff --git a/lib/license/management.rb b/lib/license/management.rb new file mode 100644 index 0000000..3b41843 --- /dev/null +++ b/lib/license/management.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +require 'pathname' +require 'yaml' +require 'license_finder' +require 'license/management/loggable' +require 'license/management/verifiable' +require 'license/management/repository' +require 'license/management/versions/base' +require 'license/management/versions/v1' +require 'license/management/json_report' +require 'license/management/version' + +module License + module Management + def self.root + Pathname.new(File.dirname(__FILE__)).join('../..') + end + end +end diff --git a/lib/license/management/json_report.rb b/lib/license/management/json_report.rb new file mode 100644 index 0000000..19a44c4 --- /dev/null +++ b/lib/license/management/json_report.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# This is a monkey patch of the JsonReport found in `license_finder` +module LicenseFinder + class JsonReport < CsvReport + DEFAULT_VERSION = '1' + VERSIONS = { + nil => ::License::Management::Versions::V1, + '' => ::License::Management::Versions::V1, + '1' => ::License::Management::Versions::V1, + '1.0' => ::License::Management::Versions::V1 + }.freeze + + def to_s + JSON.pretty_generate(version_for(report_version).to_h) + "\n" + end + + private + + def report_version + ENV.fetch('LM_REPORT_VERSION', DEFAULT_VERSION) + end + + def version_for(version) + VERSIONS.fetch(version.to_s).new(dependencies) + end + end +end diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb new file mode 100644 index 0000000..a44d45d --- /dev/null +++ b/lib/license/management/loggable.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module License + module Management + module Loggable + def logger + ::LicenseFinder::Core.default_logger + end + + def log_info(message) + logger.info(self.class, message) + end + end + end +end diff --git a/lib/license/management/repository.rb b/lib/license/management/repository.rb new file mode 100644 index 0000000..707d9f0 --- /dev/null +++ b/lib/license/management/repository.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +module License + module Management + class Repository + include Loggable + include Verifiable + + def initialize( + compatibility_path: License::Management.root.join('licenses.yml') + ) + @compatibility_data = YAML.safe_load(IO.read(compatibility_path)) + end + + def item_for(license) + id = id_for(license) + item = id ? compatibility_data['items'][id] : nil + item ? { 'id' => id }.merge(item) : generate_item_for(license) + end + + private + + attr_reader :compatibility_data + + def id_for(license) + ids = compatibility_data['ids'] + ids[license.send(:short_name)] || ids[license.url] + end + + def take_first_line_from(content) + return '' if blank?(content) + + content.split(/[\r\n]+/)[0] + end + + def generate_item_for(license) + log_info("detected unknown license named `#{license.send(:short_name)}`") + { + 'id' => 'unknown', + 'name' => take_first_line_from(license.name), + 'url' => present?(license.url) ? license.url : '' + } + end + end + end +end diff --git a/lib/license/management/verifiable.rb b/lib/license/management/verifiable.rb new file mode 100644 index 0000000..72667fb --- /dev/null +++ b/lib/license/management/verifiable.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module License + module Management + module Verifiable + def blank?(item) + item.nil? || item.empty? + end + + def present?(item) + !blank?(item) + end + end + end +end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb new file mode 100644 index 0000000..d679b3f --- /dev/null +++ b/lib/license/management/version.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module License + module Management + VERSION = '1.6.0' + end +end diff --git a/lib/license/management/versions/base.rb b/lib/license/management/versions/base.rb new file mode 100644 index 0000000..6e30e20 --- /dev/null +++ b/lib/license/management/versions/base.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module License + module Management + module Versions + class Base + include Loggable + include Verifiable + + attr_reader :dependencies, :repository + + def initialize(dependencies) + @dependencies = dependencies + @repository = License::Management::Repository.new + end + + private + + def paths_from(dependency) + return [] unless dependency.respond_to?(:aggregate_paths) + + paths = dependency.aggregate_paths + return [] if blank?(paths) + + paths.map { |x| x.gsub(Dir.pwd, '.') } + end + + def description_for(dependency) + present?(dependency.summary) ? dependency.summary : dependency.description + end + end + end + end +end diff --git a/lib/license/management/versions/v1.rb b/lib/license/management/versions/v1.rb new file mode 100644 index 0000000..d448397 --- /dev/null +++ b/lib/license/management/versions/v1.rb @@ -0,0 +1,72 @@ +# frozen_string_literal: true + +module License + module Management + module Versions + class V1 < Base + def to_h + { + licenses: license_summary, + dependencies: formatted_dependencies(dependencies) + } + end + + private + + # when a dependency has multiple licenses, this will join the licenses into a single name + # this defect was backported from the htmltojson version of this script. + def license_summary + dependencies + .map { |x| x.licenses.map { |y| best_name_for(y) }.sort.reverse.join(', ') } + .flatten + .group_by { |name| name } + .map { |(license, items)| { count: items.count, name: license } } + .sort_by { |x| [-x[:count], x[:name]] } + end + + # when a dependency has more than one license + # this method chooses one of the urls. + # to maintain backwards compatibility this bug has been carried forward. + def license_for(dependency) + name = join_license_names(dependency.licenses) + urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort + log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1 + url = urls[0] || data_for(dependency.licenses.first)['url'] + blank?(url) ? { name: name } : { name: name, url: url } + end + + def join_license_names(licenses) + licenses.map { |x| best_name_for(x) }.join(', ') + end + + def map_from_dependency(dependency) + result = { + license: license_for(dependency), + dependency: { + name: dependency.name, + url: dependency.homepage, + description: description_for(dependency), + pathes: paths_from(dependency) + } + } + result[:dependency].delete(:url) if blank?(dependency.homepage) + result + end + + def formatted_dependencies(dependencies) + dependencies + .sort_by(&:name) + .map { |x| map_from_dependency(x) } + end + + def best_name_for(license) + data_for(license)['name'] + end + + def data_for(license) + repository.item_for(license) + end + end + end + end +end diff --git a/license-management.gemspec b/license-management.gemspec new file mode 100644 index 0000000..99cb1b1 --- /dev/null +++ b/license-management.gemspec @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'license/management/version' + +Gem::Specification.new do |spec| + spec.name = 'license-management' + spec.version = License::Management::VERSION + spec.authors = ['Fabien Catteau', 'Olivier Gonzalez'] + spec.email = ['fcatteau@gitlab.com', 'ogonzalez@gitlab.com'] + + spec.summary = 'License Management job for GitLab CI.' + spec.description = 'License Management job for GitLab CI.' + spec.homepage = 'https://gitlab.com/gitlab-org/security-products/license-management' + spec.license = 'GitLab EE' + + spec.metadata['allowed_push_host'] = 'https://example.com' + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management' + spec.metadata['changelog_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management/blob/master/CHANGELOG.md' + + spec.files = Dir.chdir(File.expand_path(__dir__)) do + Dir.glob('exe/*') + Dir.glob('lib/**/**/*.{rb,yml}') + Dir.glob('*.{md,yml,json}') + end + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'license_finder', ENV.fetch('LICENSE_FINDER_VERSION', '5.6.2') +end diff --git a/licenses.yml b/licenses.yml new file mode 100644 index 0000000..8ed24b2 --- /dev/null +++ b/licenses.yml @@ -0,0 +1,82 @@ +--- +ids: + ASL, version 2: Apache-2.0 + Apache1_1: Apache-1.1 + Apache2: Apache-2.0 + Apache 2.0: Apache-2.0 + BSD: BSD-4-Clause + CC01: CC0-1.0 + EPL1: EPL-1.0 + GPLv2: GPL-2.0 + GPLv3: GPL-3.0 + ISC: ISC + LGPL 2.1: LGPL-2.1 + LGPL, version 2.1: LGPL-2.1 + LGPL2_1: LGPL-2.1 + LGPL: LGPL-3.0-only + MIT: MIT + MPL 1.1: MPL-1.1 + MPL1_1: MPL-1.1 + MPL2: MPL-2.0 + New BSD License: BSD-3-Clause + NewBSD: BSD-3-Clause + Python: Python-2.0 + Ruby: Ruby + SimplifiedBSD: BSD-2-Clause + http://www.apache.org/licenses/LICENSE-2.0: Apache-2.0 + unknown: unknown +items: + GPL-3.0: + name: GPL 3.0 + url: https://www.gnu.org/licenses/gpl-3.0.txt + BSD-2-Clause: + name: Simplified BSD + url: https://opensource.org/licenses/bsd-license + ISC: + name: ISC + url: https://en.wikipedia.org/wiki/ISC_license + Apache-1.1: + name: Apache 1.1 + url: https://www.apache.org/licenses/LICENSE-1.1.txt + MPL-2.0: + name: Mozilla Public License 2.0 + url: https://www.mozilla.org/media/MPL/2.0/index.815ca599c9df.txt + LGPL-3.0-only: + name: LGPL + url: https://www.gnu.org/licenses/lgpl.txt + CC0-1.0: + name: CC0 1.0 + url: https://creativecommons.org/publicdomain/zero/1.0/ + unknown: + name: unknown + url: '' + GPL-2.0: + name: GPL 2.0 + url: https://www.gnu.org/licenses/gpl-2.0.txt + EPL-1.0: + name: EPL 1.0 + url: https://www.eclipse.org/legal/epl-v10.html + Python-2.0: + name: Python Software Foundation License + url: https://hg.python.org/cpython/raw-file/89ce323357db/LICENSE + BSD-3-Clause: + name: New BSD License + url: https://opensource.org/licenses/BSD-3-Clause + MIT: + name: MIT + url: https://opensource.org/licenses/mit-license + LGPL-2.1: + name: LGPL 2.1 + url: https://opensource.org/licenses/LGPL-2.1 + Ruby: + name: Ruby + url: https://www.ruby-lang.org/en/about/license.txt + Apache-2.0: + name: Apache 2.0 + url: https://www.apache.org/licenses/LICENSE-2.0.txt + MPL-1.1: + name: MPL 1.1 + url: https://www.mozilla.org/en-US/MPL/1.1/ + BSD-4-Clause: + name: BSD + url: https://en.wikipedia.org/wiki/BSD_licenses#4-clause_license_.28original_.22BSD_License.22.29 @@ -115,12 +115,13 @@ case "$COMMAND" in gem install bundler # We need to install the license_finder gem into this Ruby version too. gem install license_finder -v "$LICENSE_FINDER_VERSION" + gem install --no-document /opt/license-management/*.gem fi # Ignore test and development dependencies. license_finder ignored_groups add development license_finder ignored_groups add test - bundle install --without "development test" + bundle install --without "development test" --jobs $(nproc) skip_prepare=true fi @@ -189,17 +190,24 @@ case "$COMMAND" in # Run License Finder. echo "Running license_finder $@ in $PWD" - if [ "$skip_prepare" != true ]; then prepare="--prepare" fi - license_finder report ${prepare} --format=html --save=gl-license-management-report.html + if [ "$FEATURE_RUBY_REPORT" = 'true' ]; then + echo "Preparing JSON report..." + license_management report ${prepare} --format=json --save=gl-license-management-report.json + else + echo "Preparing HTML report..." + license_finder report ${prepare} --format=html --save=gl-license-management-report.html + fi # rvm removes trap in bash: https://github.com/rvm/rvm/issues/4416 declare -f restore_lockfile > /dev/null && restore_lockfile popd > /dev/null - # Extract data from the HTML report and put it into a JSON file - node /html2json.js $APP_PATH/gl-license-management-report.html > $APP_PATH/gl-license-management-report.json + if [ -f "gl-license-management-report.html" ]; then + # Extract data from the HTML report and put it into a JSON file + node /html2json.js $APP_PATH/gl-license-management-report.html > $APP_PATH/gl-license-management-report.json + fi ;; *) diff --git a/test/results/go-modules.json b/test/results/go-modules-v1.json index b946b60..b946b60 100644 --- a/test/results/go-modules.json +++ b/test/results/go-modules-v1.json diff --git a/test/results/java-maven.json b/test/results/java-maven-v1.json index 58e9260..58e9260 100644 --- a/test/results/java-maven.json +++ b/test/results/java-maven-v1.json diff --git a/test/results/python-pip.json b/test/results/python-pip-v1.json index 3aa6f39..3aa6f39 100644 --- a/test/results/python-pip.json +++ b/test/results/python-pip-v1.json diff --git a/test/results/python3-pip.json b/test/results/python3-pip-v1.json index d98669d..d98669d 100644 --- a/test/results/python3-pip.json +++ b/test/results/python3-pip-v1.json diff --git a/test/results/ruby-bundler.json b/test/results/ruby-bundler-v1.json index c66a9a3..c66a9a3 100644 --- a/test/results/ruby-bundler.json +++ b/test/results/ruby-bundler-v1.json |