From 0f07af5b661077312192b75d1ff1a5b8f6a123aa Mon Sep 17 00:00:00 2001 From: mo khan Date: Fri, 28 Feb 2020 15:27:12 -0700 Subject: Convert to gitlab-license-scanning --- .gitignore | 1 + Gemfile.lock | 22 +++++- exe/gitlab-license-scanning | 6 ++ exe/license_management | 6 -- gitlab-license-scanning.gemspec | 33 +++++++++ lib/gitlab/license/scanning.rb | 32 +++++++++ lib/gitlab/license/scanning/loggable.rb | 21 ++++++ lib/gitlab/license/scanning/report.rb | 41 +++++++++++ lib/gitlab/license/scanning/report/base.rb | 40 +++++++++++ lib/gitlab/license/scanning/report/v1.rb | 78 ++++++++++++++++++++ lib/gitlab/license/scanning/report/v1_1.rb | 29 ++++++++ lib/gitlab/license/scanning/report/v2.rb | 49 +++++++++++++ lib/gitlab/license/scanning/repository.rb | 95 ++++++++++++++++++++++++ lib/gitlab/license/scanning/verifiable.rb | 17 +++++ lib/gitlab/license/scanning/version.rb | 9 +++ lib/license/management.rb | 29 -------- lib/license/management/loggable.rb | 19 ----- lib/license/management/report.rb | 39 ---------- lib/license/management/report/base.rb | 38 ---------- lib/license/management/report/v1.rb | 76 -------------------- lib/license/management/report/v1_1.rb | 27 ------- lib/license/management/report/v2.rb | 47 ------------ lib/license/management/repository.rb | 96 ------------------------- lib/license/management/verifiable.rb | 15 ---- lib/license/management/version.rb | 7 -- license-management.gemspec | 32 --------- spec/spec_helper.rb | 2 +- spec/unit/license/management/report/v2_spec.rb | 51 ------------- spec/unit/license/management/repository_spec.rb | 38 ---------- spec/unit/report/v2_spec.rb | 50 +++++++++++++ spec/unit/repository_spec.rb | 38 ++++++++++ 31 files changed, 560 insertions(+), 523 deletions(-) create mode 100755 exe/gitlab-license-scanning delete mode 100755 exe/license_management create mode 100644 gitlab-license-scanning.gemspec create mode 100644 lib/gitlab/license/scanning.rb create mode 100644 lib/gitlab/license/scanning/loggable.rb create mode 100644 lib/gitlab/license/scanning/report.rb create mode 100644 lib/gitlab/license/scanning/report/base.rb create mode 100644 lib/gitlab/license/scanning/report/v1.rb create mode 100644 lib/gitlab/license/scanning/report/v1_1.rb create mode 100644 lib/gitlab/license/scanning/report/v2.rb create mode 100644 lib/gitlab/license/scanning/repository.rb create mode 100644 lib/gitlab/license/scanning/verifiable.rb create mode 100644 lib/gitlab/license/scanning/version.rb delete mode 100644 lib/license/management.rb delete mode 100644 lib/license/management/loggable.rb delete mode 100644 lib/license/management/report.rb delete mode 100644 lib/license/management/report/base.rb delete mode 100644 lib/license/management/report/v1.rb delete mode 100644 lib/license/management/report/v1_1.rb delete mode 100644 lib/license/management/report/v2.rb delete mode 100644 lib/license/management/repository.rb delete mode 100644 lib/license/management/verifiable.rb delete mode 100644 lib/license/management/version.rb delete mode 100644 license-management.gemspec delete mode 100644 spec/unit/license/management/report/v2_spec.rb delete mode 100644 spec/unit/license/management/repository_spec.rb create mode 100644 spec/unit/report/v2_spec.rb create mode 100644 spec/unit/repository_spec.rb diff --git a/.gitignore b/.gitignore index 8462f59..45379c4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ tmp Dockerfile.env +*.log diff --git a/Gemfile.lock b/Gemfile.lock index 740baeb..24412f7 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,9 @@ PATH remote: . specs: - license-management (2.7.0) + gitlab-license-scanning (2.7.0) license_finder (~> 6.0.0) + spandx (~> 0.1) GEM remote: https://rubygems.org/ @@ -19,6 +20,11 @@ GEM toml (= 0.2.0) with_env (= 1.1.0) xml-simple + mini_portile2 (2.4.0) + msgpack (1.3.3) + net-hippie (0.3.2) + nokogiri (1.10.8) + mini_portile2 (~> 2.4.0) parslet (1.8.2) public_suffix (4.0.3) rspec (3.9.0) @@ -35,6 +41,18 @@ GEM rspec-support (~> 3.9.0) rspec-support (3.9.2) rubyzip (2.2.0) + spandx (0.5.0) + addressable (~> 2.7) + bundler (>= 1.16, < 3.0.0) + net-hippie (~> 0.3) + nokogiri (~> 1.10) + spandx-rubygems (~> 0.1) + thor + spandx-rubygems (0.1.3) + msgpack (~> 1.3) + net-hippie (~> 0.3) + nokogiri (~> 1.10) + thor thor (1.0.1) toml (0.2.0) parslet (~> 1.8.0) @@ -45,8 +63,8 @@ PLATFORMS ruby DEPENDENCIES + gitlab-license-scanning! json-schema (~> 2.8) - license-management! rspec (~> 3.9) BUNDLED WITH diff --git a/exe/gitlab-license-scanning b/exe/gitlab-license-scanning new file mode 100755 index 0000000..2fdc05f --- /dev/null +++ b/exe/gitlab-license-scanning @@ -0,0 +1,6 @@ +#!/usr/bin/env ruby +# frozen_string_literal: true + +require 'gitlab/license/scanning' + +LicenseFinder::CLI::Main.start(ARGV) diff --git a/exe/license_management b/exe/license_management deleted file mode 100755 index 33e110e..0000000 --- a/exe/license_management +++ /dev/null @@ -1,6 +0,0 @@ -#!/usr/bin/env ruby -# frozen_string_literal: true - -require 'license/management' - -LicenseFinder::CLI::Main.start(ARGV) diff --git a/gitlab-license-scanning.gemspec b/gitlab-license-scanning.gemspec new file mode 100644 index 0000000..681b658 --- /dev/null +++ b/gitlab-license-scanning.gemspec @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +lib = File.expand_path('lib', __dir__) +$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) +require 'gitlab/license/scanning/version' + +Gem::Specification.new do |spec| + spec.name = 'gitlab-license-scanning' + spec.version = GitLab::License::Scanning::VERSION + spec.authors = ['Fabien Catteau', 'Olivier Gonzalez', 'mo khan'] + spec.email = ['fcatteau@gitlab.com', 'ogonzalez@gitlab.com', 'mkhan@gitlab.com'] + + spec.summary = 'License Scanning job for GitLab CI.' + spec.description = 'License Scanning job for GitLab CI.' + spec.homepage = 'https://gitlab.com/gitlab-org/security-products/license-management' + spec.license = 'Nonstandard' + + spec.metadata['homepage_uri'] = spec.homepage + spec.metadata['source_code_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management' + spec.metadata['changelog_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management/blob/master/CHANGELOG.md' + + spec.files = Dir.chdir(File.expand_path(__dir__)) do + Dir.glob('exe/*') + Dir.glob('lib/**/**/*.{rb,yml}') + Dir.glob('*.{md,yml,json}') + end + spec.bindir = 'exe' + spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } + spec.require_paths = ['lib'] + + spec.add_dependency 'license_finder', '~> 6.0.0' + spec.add_dependency 'spandx', '~> 0.1' + spec.add_development_dependency 'json-schema', '~> 2.8' + spec.add_development_dependency 'rspec', '~> 3.9' +end diff --git a/lib/gitlab/license/scanning.rb b/lib/gitlab/license/scanning.rb new file mode 100644 index 0000000..f8fa375 --- /dev/null +++ b/lib/gitlab/license/scanning.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +require 'pathname' +require 'yaml' +require 'json' +require 'license_finder' +require 'spandx' +require 'gitlab/license/scanning/loggable' +require 'gitlab/license/scanning/verifiable' +require 'gitlab/license/scanning/repository' +require 'gitlab/license/scanning/report' +require 'gitlab/license/scanning/version' + +# This applies a monkey patch to the JsonReport found in the `license_finder` gem. +LicenseFinder::JsonReport.prepend(GitLab::License::Scanning::Report) + +module GitLab + module License + module Scanning + def self.root + Pathname.new(File.dirname(__FILE__)).join('../../..') + end + + def self.http + @http ||= Net::Hippie::Client.new.tap do |client| + client.logger = ::Logger.new('http.log') + client.follow_redirects = 3 + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/loggable.rb b/lib/gitlab/license/scanning/loggable.rb new file mode 100644 index 0000000..e101e82 --- /dev/null +++ b/lib/gitlab/license/scanning/loggable.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + module Loggable + def logger + ::LicenseFinder::Core.default_logger + end + + def log_info(message) + logger.info(self.class, message) + end + + def log_error(message) + logger.info(self.class, message, color: :red) + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/report.rb b/lib/gitlab/license/scanning/report.rb new file mode 100644 index 0000000..102dea6 --- /dev/null +++ b/lib/gitlab/license/scanning/report.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'gitlab/license/scanning/report/base' +require 'gitlab/license/scanning/report/v1' +require 'gitlab/license/scanning/report/v1_1' +require 'gitlab/license/scanning/report/v2' + +module GitLab + module License + module Scanning + module Report + DEFAULT_VERSION = '1' + VERSIONS = { + nil => V1, + '' => V1, + '1' => V1, + '1.0' => V1, + '1.1' => V1_1, + '2' => V2, + '2.0' => V2 + }.freeze + + # This method overrides the method defined in `LicenseFinder::JsonReport` to + # allow us to generate a custom json report. + def to_s + JSON.pretty_generate(version_for(report_version).to_h) + "\n" + end + + private + + def report_version + ENV.fetch('LM_REPORT_VERSION', DEFAULT_VERSION) + end + + def version_for(version) + VERSIONS.fetch(version.to_s).new(dependencies) + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/report/base.rb b/lib/gitlab/license/scanning/report/base.rb new file mode 100644 index 0000000..b8d0219 --- /dev/null +++ b/lib/gitlab/license/scanning/report/base.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + module Report + class Base + include Loggable + include Verifiable + + attr_reader :dependencies, :repository + + def initialize(dependencies) + @dependencies = dependencies.sort + @repository = Repository.new + end + + def to_h + raise NotImplementedError + end + + private + + def paths_from(dependency) + return [] unless dependency.respond_to?(:aggregate_paths) + + paths = dependency.aggregate_paths + return [] if blank?(paths) + + paths.map { |x| x.gsub(Dir.pwd, '.') } + end + + def description_for(dependency) + present?(dependency.summary) ? dependency.summary : dependency.description + end + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/report/v1.rb b/lib/gitlab/license/scanning/report/v1.rb new file mode 100644 index 0000000..faece8d --- /dev/null +++ b/lib/gitlab/license/scanning/report/v1.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + module Report + class V1 < Base + def to_h + { + licenses: license_summary, + dependencies: formatted_dependencies(dependencies) + } + end + + private + + # when a dependency has multiple licenses, this will join the licenses into a single name + # this defect was backported from the [html2json](https://gitlab.com/gitlab-org/security-products/license-management/blob/7f175952a5a047d785b5ea72c15a10642523c62a/html2json.js) version of this script. + def license_summary + dependencies + .map { |dependency| join_license_names(dependency.licenses) } + .flatten + .group_by { |name| name } + .map { |license, items| { count: items.count, name: license } } + .sort_by { |hash| [-hash[:count], hash[:name]] } + end + + # when a dependency has more than one license + # this method chooses one of the urls. + # to maintain backwards compatibility this bug has been carried forward. + def license_for(dependency) + license = { name: join_license_names(dependency.licenses) } + + urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort + log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1 + url = urls[0] || license_data(dependency.licenses.first)['url'] + + license[:url] = url if present?(url) + license + end + + def join_license_names(licenses) + licenses.map { |x| best_name_for(x) }.sort.reverse.join(', ') + end + + def map_from_dependency(dependency) + result = { + license: license_for(dependency), + dependency: { + name: dependency.name, + url: dependency.homepage, + description: description_for(dependency), + pathes: paths_from(dependency) + } + } + result[:dependency].delete(:url) if blank?(dependency.homepage) + result + end + + def formatted_dependencies(dependencies) + dependencies.map { |x| map_from_dependency(x) } + end + + def best_name_for(license) + license_data(license)['name'] + end + + def license_data(license) + { + 'name' => license.name.split(/[\r\n]+/)[0], + 'url' => license.url || '' + } + end + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/report/v1_1.rb b/lib/gitlab/license/scanning/report/v1_1.rb new file mode 100644 index 0000000..a40f87f --- /dev/null +++ b/lib/gitlab/license/scanning/report/v1_1.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + module Report + class V1_1 < V1 + def to_h + { version: '1.1' }.merge(super) + end + + private + + def map_from_dependency(dependency) + licenses = dependency.licenses.sort_by(&:name).map do |license| + item = license_data(license) + { + name: item['name'], + url: item.fetch('url', '') + } + end + + { licenses: licenses }.merge(super) + end + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/report/v2.rb b/lib/gitlab/license/scanning/report/v2.rb new file mode 100644 index 0000000..db7d7f0 --- /dev/null +++ b/lib/gitlab/license/scanning/report/v2.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + module Report + class V2 < Base + def to_h + { + version: '2.0', + licenses: license_summary, + dependencies: dependencies.map { |x| map_from(x) } + } + end + + private + + def all_licenses + dependencies.map { |x| x.licenses.to_a }.flatten + end + + def license_summary + all_licenses + .group_by { |x| data_for(x)['name'] } + .sort_by { |x, y| [-y.size, x] } + .map { |_name, items| data_for(items[0]).merge(count: items.count) } + end + + def data_for(license) + repository.item_for(license) + end + + def map_from(dependency) + licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort + log_info [dependency.name, dependency.version, licenses].inspect + + { + name: dependency.name, + url: dependency.homepage, + description: description_for(dependency), + paths: paths_from(dependency), + licenses: licenses + } + end + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/repository.rb b/lib/gitlab/license/scanning/repository.rb new file mode 100644 index 0000000..13b975a --- /dev/null +++ b/lib/gitlab/license/scanning/repository.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + class Repository + include Loggable + include Verifiable + KNOWN_SOURCES = [ + 'licenses.nuget.org', + 'opensource.org', + 'www.opensource.org', + ].freeze + + def initialize(compatibility_path: GitLab::License::Scanning.root.join('normalized-licenses.yml')) + @compatibility_data = YAML.safe_load(IO.read(compatibility_path)) + @catalogue = Spandx::Catalogue.latest + end + + def item_for(license) + spdx_data_for(id_for(license)) || + spdx_data_for(license.send(:short_name)) || + generate_item_for(license) + end + + private + + attr_reader :catalogue, :compatibility_data + + def spdx_data_for(id) + return if blank?(id) + + data = catalogue[id] + if data + { + 'id' => data.id, + 'name' => data.name, + 'url' => data.see_also[-1] + } + end + end + + def id_for(license) + ids = compatibility_data['ids'] + ids[license.send(:short_name)] || + ids[license.url] || + known_sources(license.send(:short_name)) || + known_sources(license.url) + end + + # When `license_finder` is unable to determine the license it will use the full + # content of the file as the name of the license. This method shrinks that name + # down to just take the first line of the file. + def take_first_line_from(content) + return '' if blank?(content) + + content.split(/[\r\n]+/)[0] + end + + def generate_item_for(license) + log_info("Detected unknown license `#{license.send(:short_name)}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.") + name = take_first_line_from(license.name) + { + 'id' => name.downcase, + 'name' => name, + 'url' => present?(license.url) ? license.url : '' + } + end + + def load_spdx_data_from(path) + content = IO.read(path) + json = JSON.parse(content) + licenses = json['licenses'] + + licenses.inject({}) do |memo, license| + memo[license['licenseId']] = license + memo + end + end + + def known_sources(url) + return if blank?(url) + return unless url =~ /\A#{::URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/ + + uri = URI.parse(url) + return unless KNOWN_SOURCES.include?(uri.host.downcase) + uri.path.split('/')[-1] + rescue => error + log_info(error) + nil + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/verifiable.rb b/lib/gitlab/license/scanning/verifiable.rb new file mode 100644 index 0000000..1d24c53 --- /dev/null +++ b/lib/gitlab/license/scanning/verifiable.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + module Verifiable + def blank?(item) + item.nil? || item.empty? + end + + def present?(item) + !blank?(item) + end + end + end + end +end diff --git a/lib/gitlab/license/scanning/version.rb b/lib/gitlab/license/scanning/version.rb new file mode 100644 index 0000000..545efc1 --- /dev/null +++ b/lib/gitlab/license/scanning/version.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module GitLab + module License + module Scanning + VERSION = '2.7.0' + end + end +end diff --git a/lib/license/management.rb b/lib/license/management.rb deleted file mode 100644 index 37fe1a0..0000000 --- a/lib/license/management.rb +++ /dev/null @@ -1,29 +0,0 @@ -# frozen_string_literal: true - -require 'pathname' -require 'yaml' -require 'json' -require 'license_finder' -require 'license/management/loggable' -require 'license/management/verifiable' -require 'license/management/repository' -require 'license/management/report' -require 'license/management/version' - -# This applies a monkey patch to the JsonReport found in the `license_finder` gem. -LicenseFinder::JsonReport.prepend(License::Management::Report) - -module License - module Management - def self.root - Pathname.new(File.dirname(__FILE__)).join('../..') - end - - def self.http - @http ||= Net::Hippie::Client.new.tap do |client| - client.logger = ::Logger.new('http.log') - client.follow_redirects = 3 - end - end - end -end diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb deleted file mode 100644 index 165d8ca..0000000 --- a/lib/license/management/loggable.rb +++ /dev/null @@ -1,19 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - module Loggable - def logger - ::LicenseFinder::Core.default_logger - end - - def log_info(message) - logger.info(self.class, message) - end - - def log_error(message) - logger.info(self.class, message, color: :red) - end - end - end -end diff --git a/lib/license/management/report.rb b/lib/license/management/report.rb deleted file mode 100644 index aa3111d..0000000 --- a/lib/license/management/report.rb +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -require 'license/management/report/base' -require 'license/management/report/v1' -require 'license/management/report/v1_1' -require 'license/management/report/v2' - -module License - module Management - module Report - DEFAULT_VERSION = '1' - VERSIONS = { - nil => V1, - '' => V1, - '1' => V1, - '1.0' => V1, - '1.1' => V1_1, - '2' => V2, - '2.0' => V2 - }.freeze - - # This method overrides the method defined in `LicenseFinder::JsonReport` to - # allow us to generate a custom json report. - def to_s - JSON.pretty_generate(version_for(report_version).to_h) + "\n" - end - - private - - def report_version - ENV.fetch('LM_REPORT_VERSION', DEFAULT_VERSION) - end - - def version_for(version) - VERSIONS.fetch(version.to_s).new(dependencies) - end - end - end -end diff --git a/lib/license/management/report/base.rb b/lib/license/management/report/base.rb deleted file mode 100644 index 2d49341..0000000 --- a/lib/license/management/report/base.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - module Report - class Base - include Loggable - include Verifiable - - attr_reader :dependencies, :repository - - def initialize(dependencies) - @dependencies = dependencies.sort - @repository = License::Management::Repository.new - end - - def to_h - raise NotImplementedError - end - - private - - def paths_from(dependency) - return [] unless dependency.respond_to?(:aggregate_paths) - - paths = dependency.aggregate_paths - return [] if blank?(paths) - - paths.map { |x| x.gsub(Dir.pwd, '.') } - end - - def description_for(dependency) - present?(dependency.summary) ? dependency.summary : dependency.description - end - end - end - end -end diff --git a/lib/license/management/report/v1.rb b/lib/license/management/report/v1.rb deleted file mode 100644 index 49423c6..0000000 --- a/lib/license/management/report/v1.rb +++ /dev/null @@ -1,76 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - module Report - class V1 < Base - def to_h - { - licenses: license_summary, - dependencies: formatted_dependencies(dependencies) - } - end - - private - - # when a dependency has multiple licenses, this will join the licenses into a single name - # this defect was backported from the [html2json](https://gitlab.com/gitlab-org/security-products/license-management/blob/7f175952a5a047d785b5ea72c15a10642523c62a/html2json.js) version of this script. - def license_summary - dependencies - .map { |dependency| join_license_names(dependency.licenses) } - .flatten - .group_by { |name| name } - .map { |license, items| { count: items.count, name: license } } - .sort_by { |hash| [-hash[:count], hash[:name]] } - end - - # when a dependency has more than one license - # this method chooses one of the urls. - # to maintain backwards compatibility this bug has been carried forward. - def license_for(dependency) - license = { name: join_license_names(dependency.licenses) } - - urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort - log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1 - url = urls[0] || license_data(dependency.licenses.first)['url'] - - license[:url] = url if present?(url) - license - end - - def join_license_names(licenses) - licenses.map { |x| best_name_for(x) }.sort.reverse.join(', ') - end - - def map_from_dependency(dependency) - result = { - license: license_for(dependency), - dependency: { - name: dependency.name, - url: dependency.homepage, - description: description_for(dependency), - pathes: paths_from(dependency) - } - } - result[:dependency].delete(:url) if blank?(dependency.homepage) - result - end - - def formatted_dependencies(dependencies) - dependencies.map { |x| map_from_dependency(x) } - end - - def best_name_for(license) - license_data(license)['name'] - end - - def license_data(license) - { - 'name' => license.name.split(/[\r\n]+/)[0], - 'url' => license.url || '' - } - end - end - end - end -end diff --git a/lib/license/management/report/v1_1.rb b/lib/license/management/report/v1_1.rb deleted file mode 100644 index 1186e2a..0000000 --- a/lib/license/management/report/v1_1.rb +++ /dev/null @@ -1,27 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - module Report - class V1_1 < V1 - def to_h - { version: '1.1' }.merge(super) - end - - private - - def map_from_dependency(dependency) - licenses = dependency.licenses.sort_by(&:name).map do |license| - item = license_data(license) - { - name: item['name'], - url: item.fetch('url', '') - } - end - - { licenses: licenses }.merge(super) - end - end - end - end -end diff --git a/lib/license/management/report/v2.rb b/lib/license/management/report/v2.rb deleted file mode 100644 index 6ab6b99..0000000 --- a/lib/license/management/report/v2.rb +++ /dev/null @@ -1,47 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - module Report - class V2 < Base - def to_h - { - version: '2.0', - licenses: license_summary, - dependencies: dependencies.map { |x| map_from(x) } - } - end - - private - - def all_licenses - dependencies.map { |x| x.licenses.to_a }.flatten - end - - def license_summary - all_licenses - .group_by { |x| data_for(x)['name'] } - .sort_by { |x, y| [-y.size, x] } - .map { |_name, items| data_for(items[0]).merge(count: items.count) } - end - - def data_for(license) - repository.item_for(license) - end - - def map_from(dependency) - licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort - log_info [dependency.name, dependency.version, licenses].inspect - - { - name: dependency.name, - url: dependency.homepage, - description: description_for(dependency), - paths: paths_from(dependency), - licenses: licenses - } - end - end - end - end -end diff --git a/lib/license/management/repository.rb b/lib/license/management/repository.rb deleted file mode 100644 index 9490af2..0000000 --- a/lib/license/management/repository.rb +++ /dev/null @@ -1,96 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - class Repository - include Loggable - include Verifiable - KNOWN_SOURCES = [ - 'licenses.nuget.org', - 'opensource.org', - 'www.opensource.org', - ].freeze - - def initialize( - compatibility_path: License::Management.root.join('normalized-licenses.yml'), - spdx_path: License::Management.root.join('spdx-licenses.json') - ) - @compatibility_data = YAML.safe_load(IO.read(compatibility_path)) - @spdx_data = load_spdx_data_from(spdx_path) - end - - def item_for(license) - spdx_data_for(id_for(license)) || - spdx_data_for(license.send(:short_name)) || - generate_item_for(license) - end - - private - - attr_reader :spdx_data, :compatibility_data - - def spdx_data_for(id) - return if blank?(id) - - data = spdx_data[id] - if data - { - 'id' => data['licenseId'], - 'name' => data['name'], - 'url' => data['seeAlso'][-1] - } - end - end - - def id_for(license) - ids = compatibility_data['ids'] - ids[license.send(:short_name)] || - ids[license.url] || - known_sources(license.send(:short_name)) || - known_sources(license.url) - end - - # When `license_finder` is unable to determine the license it will use the full - # content of the file as the name of the license. This method shrinks that name - # down to just take the first line of the file. - def take_first_line_from(content) - return '' if blank?(content) - - content.split(/[\r\n]+/)[0] - end - - def generate_item_for(license) - log_info("Detected unknown license `#{license.send(:short_name)}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.") - name = take_first_line_from(license.name) - { - 'id' => name.downcase, - 'name' => name, - 'url' => present?(license.url) ? license.url : '' - } - end - - def load_spdx_data_from(path) - content = IO.read(path) - json = JSON.parse(content) - licenses = json['licenses'] - - licenses.inject({}) do |memo, license| - memo[license['licenseId']] = license - memo - end - end - - def known_sources(url) - return if blank?(url) - return unless url =~ /\A#{::URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/ - - uri = URI.parse(url) - return unless KNOWN_SOURCES.include?(uri.host.downcase) - uri.path.split('/')[-1] - rescue => error - log_info(error) - nil - end - end - end -end diff --git a/lib/license/management/verifiable.rb b/lib/license/management/verifiable.rb deleted file mode 100644 index 72667fb..0000000 --- a/lib/license/management/verifiable.rb +++ /dev/null @@ -1,15 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - module Verifiable - def blank?(item) - item.nil? || item.empty? - end - - def present?(item) - !blank?(item) - end - end - end -end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb deleted file mode 100644 index 7a3cc03..0000000 --- a/lib/license/management/version.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - VERSION = '2.7.0' - end -end diff --git a/license-management.gemspec b/license-management.gemspec deleted file mode 100644 index 8c11525..0000000 --- a/license-management.gemspec +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -lib = File.expand_path('lib', __dir__) -$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'license/management/version' - -Gem::Specification.new do |spec| - spec.name = 'gitlab-license-scanning' - spec.version = License::Management::VERSION - spec.authors = ['Fabien Catteau', 'Olivier Gonzalez', 'mo khan'] - spec.email = ['fcatteau@gitlab.com', 'ogonzalez@gitlab.com', 'mkhan@gitlab.com'] - - spec.summary = 'License Scanning job for GitLab CI.' - spec.description = 'License Scanning job for GitLab CI.' - spec.homepage = 'https://gitlab.com/gitlab-org/security-products/license-management' - spec.license = 'Nonstandard' - - spec.metadata['homepage_uri'] = spec.homepage - spec.metadata['source_code_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management' - spec.metadata['changelog_uri'] = 'https://gitlab.com/gitlab-org/security-products/license-management/blob/master/CHANGELOG.md' - - spec.files = Dir.chdir(File.expand_path(__dir__)) do - Dir.glob('exe/*') + Dir.glob('lib/**/**/*.{rb,yml}') + Dir.glob('*.{md,yml,json}') - end - spec.bindir = 'exe' - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = ['lib'] - - spec.add_dependency 'license_finder', '~> 6.0.0' - spec.add_development_dependency 'json-schema', '~> 2.8' - spec.add_development_dependency 'rspec', '~> 3.9' -end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 15c9dad..1e48a8f 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,4 +1,4 @@ -require 'license/management' +require 'gitlab/license/scanning' require 'json' require 'securerandom' require 'json-schema' diff --git a/spec/unit/license/management/report/v2_spec.rb b/spec/unit/license/management/report/v2_spec.rb deleted file mode 100644 index 4da973c..0000000 --- a/spec/unit/license/management/report/v2_spec.rb +++ /dev/null @@ -1,51 +0,0 @@ -RSpec.describe License::Management::Report::V2 do - describe "#to_h" do - { - 'AGPL-1.0' => 'AGPL-1.0', - 'AGPL-3.0' => 'AGPL-3.0', - 'Apache 2.0' => 'Apache-2.0', - 'Artistic-2.0' => 'Artistic-2.0', - 'BSD' => 'BSD-4-Clause', - 'CC0 1.0 Universal' => 'CC0-1.0', - 'CDDL-1.0' => 'CDDL-1.0', - 'CDDL-1.1' => 'CDDL-1.1', - 'EPL-1.0' => 'EPL-1.0', - 'EPL-2.0' => 'EPL-2.0', - 'GPLv2' => 'GPL-2.0', - 'GPLv3' => 'GPL-3.0', - 'ISC' => 'ISC', - 'LGPL' => 'LGPL-3.0-only', - 'LGPL-2.1' => 'LGPL-2.1', - 'MIT' => 'MIT', - 'Mozilla Public License 2.0' => 'MPL-2.0', - 'MS-PL' => 'MS-PL', - 'MS-RL' => 'MS-RL', - 'New BSD' => 'BSD-3-Clause', - 'Python Software Foundation License' => 'Python-2.0', - 'ruby' => 'Ruby', - 'Simplified BSD' => 'BSD-2-Clause', - 'WTFPL' => 'WTFPL', - 'Zlib' => 'Zlib' - }.each do |old_name, spdx_id| - context "when mapping the legacy license name #{old_name}" do - subject { described_class.new([dependency]) } - - let(:license) { LicenseFinder::License.new(short_name: old_name, matcher: LicenseFinder::License::NoneMatcher.new, url: nil) } - let(:dependency) { instance_double(LicenseFinder::Package, name: 'x', summary: '', description: '', homepage: '', licenses: [license]).as_null_object } - let(:result) { subject.to_h } - - specify { expect(result[:version]).to eq('2.0') } - specify { expect(result[:licenses].count).to be(1) } - specify { expect(result[:licenses][0]['id']).to eq(spdx_id) } - end - end - - context "when choosing an appropriate url for a license" do - subject { described_class.new([dependency]) } - let(:license) { LicenseFinder::License.new(short_name: 'MIT', matcher: LicenseFinder::License::NoneMatcher.new, url: nil) } - let(:dependency) { instance_double(LicenseFinder::Package, name: 'x', summary: '', description: '', homepage: '', licenses: [license]).as_null_object } - - specify { expect(subject.to_h[:licenses][0]['url']).to eql('https://opensource.org/licenses/MIT') } - end - end -end diff --git a/spec/unit/license/management/repository_spec.rb b/spec/unit/license/management/repository_spec.rb deleted file mode 100644 index 6ebc09e..0000000 --- a/spec/unit/license/management/repository_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -RSpec.describe License::Management::Repository do - describe "#item_for" do - let(:spdx_licenses) { JSON.parse(IO.read('spdx-licenses.json'))['licenses'] } - - context "when mapping a license that refers to opensource.org" do - it 'parses the SPDX id from the url' do - spdx_licenses.each do |license| - spdx_id = license['licenseId'] - url = "https://opensource.org/licenses/#{spdx_id}" - license = LicenseFinder::License.new(short_name: url, matcher: LicenseFinder::License::NoneMatcher.new, url: url) - expect(subject.item_for(license)['id']).to eql(spdx_id) - end - end - end - - context "when mapping a license that refers to nuget.org" do - it 'parses the SPDX id from the url' do - spdx_licenses.each do |license| - spdx_id = license['licenseId'] - url = "https://licenses.nuget.org/#{spdx_id}" - license = LicenseFinder::License.new(short_name: url, matcher: LicenseFinder::License::NoneMatcher.new, url: url) - expect(subject.item_for(license)['id']).to eql(spdx_id) - end - end - end - - [ - ['Apache License v2.0', 'Apache-2.0'], - ].each do |short_name, spdx_id| - context "when mapping a `#{short_name}` license" do - let(:license) { LicenseFinder::License.new(short_name: short_name, matcher: LicenseFinder::License::NoneMatcher.new, url: nil) } - let(:dependency) { double(name: 'x', summary: '', description: '', homepage: '', licenses: [license]) } - - it { expect(subject.item_for(license)['id']).to eql(spdx_id) } - end - end - end -end diff --git a/spec/unit/report/v2_spec.rb b/spec/unit/report/v2_spec.rb new file mode 100644 index 0000000..7bbbc3d --- /dev/null +++ b/spec/unit/report/v2_spec.rb @@ -0,0 +1,50 @@ +RSpec.describe GitLab::License::Scanning::Report::V2 do + describe "#to_h" do + { + 'AGPL-1.0' => 'AGPL-1.0', + 'AGPL-3.0' => 'AGPL-3.0', + 'Apache 2.0' => 'Apache-2.0', + 'Artistic-2.0' => 'Artistic-2.0', + 'CC0 1.0 Universal' => 'CC0-1.0', + 'CDDL-1.0' => 'CDDL-1.0', + 'CDDL-1.1' => 'CDDL-1.1', + 'EPL-1.0' => 'EPL-1.0', + 'EPL-2.0' => 'EPL-2.0', + 'GPLv2' => 'GPL-2.0', + 'GPLv3' => 'GPL-3.0', + 'ISC' => 'ISC', + 'LGPL' => 'LGPL-3.0-only', + 'LGPL-2.1' => 'LGPL-2.1', + 'MIT' => 'MIT', + 'Mozilla Public License 2.0' => 'MPL-2.0', + 'MS-PL' => 'MS-PL', + 'MS-RL' => 'MS-RL', + 'New BSD' => 'BSD-3-Clause', + 'Python Software Foundation License' => 'Python-2.0', + 'ruby' => 'Ruby', + 'Simplified BSD' => 'BSD-2-Clause', + 'WTFPL' => 'WTFPL', + 'Zlib' => 'Zlib' + }.each do |old_name, spdx_id| + context "when mapping the legacy license name #{old_name}" do + subject { described_class.new([dependency]) } + + let(:license) { LicenseFinder::License.new(short_name: old_name, matcher: LicenseFinder::License::NoneMatcher.new, url: nil) } + let(:dependency) { instance_double(LicenseFinder::Package, name: 'x', summary: '', description: '', homepage: '', licenses: [license]).as_null_object } + let(:result) { subject.to_h } + + specify { expect(result[:version]).to eq('2.0') } + specify { expect(result[:licenses].count).to be(1) } + specify { expect(result[:licenses][0]['id']).to eq(spdx_id) } + end + end + + context "when choosing an appropriate url for a license" do + subject { described_class.new([dependency]) } + let(:license) { LicenseFinder::License.new(short_name: 'MIT', matcher: LicenseFinder::License::NoneMatcher.new, url: nil) } + let(:dependency) { instance_double(LicenseFinder::Package, name: 'x', summary: '', description: '', homepage: '', licenses: [license]).as_null_object } + + specify { expect(subject.to_h[:licenses][0]['url']).to eql('https://opensource.org/licenses/MIT') } + end + end +end diff --git a/spec/unit/repository_spec.rb b/spec/unit/repository_spec.rb new file mode 100644 index 0000000..fa00297 --- /dev/null +++ b/spec/unit/repository_spec.rb @@ -0,0 +1,38 @@ +RSpec.describe GitLab::License::Scanning::Repository do + describe "#item_for" do + let(:spdx_licenses) { Spandx::Catalogue.latest } + + context "when mapping a license that refers to opensource.org" do + it 'parses the SPDX id from the url' do + spdx_licenses.each do |license| + spdx_id = license.id + url = "https://opensource.org/licenses/#{spdx_id}" + license = LicenseFinder::License.new(short_name: url, matcher: LicenseFinder::License::NoneMatcher.new, url: url) + expect(subject.item_for(license)['id']).to eql(spdx_id) + end + end + end + + context "when mapping a license that refers to nuget.org" do + it 'parses the SPDX id from the url' do + spdx_licenses.each do |license| + spdx_id = license.id + url = "https://licenses.nuget.org/#{spdx_id}" + license = LicenseFinder::License.new(short_name: url, matcher: LicenseFinder::License::NoneMatcher.new, url: url) + expect(subject.item_for(license)['id']).to eql(spdx_id) + end + end + end + + [ + ['Apache License v2.0', 'Apache-2.0'], + ].each do |short_name, spdx_id| + context "when mapping a `#{short_name}` license" do + let(:license) { LicenseFinder::License.new(short_name: short_name, matcher: LicenseFinder::License::NoneMatcher.new, url: nil) } + let(:dependency) { double(name: 'x', summary: '', description: '', homepage: '', licenses: [license]) } + + it { expect(subject.item_for(license)['id']).to eql(spdx_id) } + end + end + end +end -- cgit v1.2.3