summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormo khan <mo.khan@gmail.com>2020-02-28 15:27:12 -0700
committermo khan <mo.khan@gmail.com>2020-02-28 15:27:12 -0700
commit0f07af5b661077312192b75d1ff1a5b8f6a123aa (patch)
tree94a6b204019a919637d616b113367f485f08770c
parentec5d95953b614de3124bd733dfc739da1383130e (diff)
Convert to gitlab-license-scanning
-rw-r--r--.gitignore1
-rw-r--r--Gemfile.lock22
-rwxr-xr-xexe/gitlab-license-scanning (renamed from exe/license_management)2
-rw-r--r--gitlab-license-scanning.gemspec (renamed from license-management.gemspec)5
-rw-r--r--lib/gitlab/license/scanning.rb32
-rw-r--r--lib/gitlab/license/scanning/loggable.rb21
-rw-r--r--lib/gitlab/license/scanning/report.rb41
-rw-r--r--lib/gitlab/license/scanning/report/base.rb40
-rw-r--r--lib/gitlab/license/scanning/report/v1.rb78
-rw-r--r--lib/gitlab/license/scanning/report/v1_1.rb29
-rw-r--r--lib/gitlab/license/scanning/report/v2.rb49
-rw-r--r--lib/gitlab/license/scanning/repository.rb95
-rw-r--r--lib/gitlab/license/scanning/verifiable.rb17
-rw-r--r--lib/gitlab/license/scanning/version.rb9
-rw-r--r--lib/license/management.rb29
-rw-r--r--lib/license/management/loggable.rb19
-rw-r--r--lib/license/management/report.rb39
-rw-r--r--lib/license/management/report/base.rb38
-rw-r--r--lib/license/management/report/v1.rb76
-rw-r--r--lib/license/management/report/v1_1.rb27
-rw-r--r--lib/license/management/report/v2.rb47
-rw-r--r--lib/license/management/repository.rb96
-rw-r--r--lib/license/management/verifiable.rb15
-rw-r--r--lib/license/management/version.rb7
-rw-r--r--spec/spec_helper.rb2
-rw-r--r--spec/unit/report/v2_spec.rb (renamed from spec/unit/license/management/report/v2_spec.rb)3
-rw-r--r--spec/unit/repository_spec.rb (renamed from spec/unit/license/management/repository_spec.rb)8
27 files changed, 442 insertions, 405 deletions
diff --git a/.gitignore b/.gitignore
index 8462f59..45379c4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,3 @@
tmp
Dockerfile.env
+*.log
diff --git a/Gemfile.lock b/Gemfile.lock
index 740baeb..24412f7 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -1,8 +1,9 @@
PATH
remote: .
specs:
- license-management (2.7.0)
+ gitlab-license-scanning (2.7.0)
license_finder (~> 6.0.0)
+ spandx (~> 0.1)
GEM
remote: https://rubygems.org/
@@ -19,6 +20,11 @@ GEM
toml (= 0.2.0)
with_env (= 1.1.0)
xml-simple
+ mini_portile2 (2.4.0)
+ msgpack (1.3.3)
+ net-hippie (0.3.2)
+ nokogiri (1.10.8)
+ mini_portile2 (~> 2.4.0)
parslet (1.8.2)
public_suffix (4.0.3)
rspec (3.9.0)
@@ -35,6 +41,18 @@ GEM
rspec-support (~> 3.9.0)
rspec-support (3.9.2)
rubyzip (2.2.0)
+ spandx (0.5.0)
+ addressable (~> 2.7)
+ bundler (>= 1.16, < 3.0.0)
+ net-hippie (~> 0.3)
+ nokogiri (~> 1.10)
+ spandx-rubygems (~> 0.1)
+ thor
+ spandx-rubygems (0.1.3)
+ msgpack (~> 1.3)
+ net-hippie (~> 0.3)
+ nokogiri (~> 1.10)
+ thor
thor (1.0.1)
toml (0.2.0)
parslet (~> 1.8.0)
@@ -45,8 +63,8 @@ PLATFORMS
ruby
DEPENDENCIES
+ gitlab-license-scanning!
json-schema (~> 2.8)
- license-management!
rspec (~> 3.9)
BUNDLED WITH
diff --git a/exe/license_management b/exe/gitlab-license-scanning
index 33e110e..2fdc05f 100755
--- a/exe/license_management
+++ b/exe/gitlab-license-scanning
@@ -1,6 +1,6 @@
#!/usr/bin/env ruby
# frozen_string_literal: true
-require 'license/management'
+require 'gitlab/license/scanning'
LicenseFinder::CLI::Main.start(ARGV)
diff --git a/license-management.gemspec b/gitlab-license-scanning.gemspec
index 8c11525..681b658 100644
--- a/license-management.gemspec
+++ b/gitlab-license-scanning.gemspec
@@ -2,11 +2,11 @@
lib = File.expand_path('lib', __dir__)
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'license/management/version'
+require 'gitlab/license/scanning/version'
Gem::Specification.new do |spec|
spec.name = 'gitlab-license-scanning'
- spec.version = License::Management::VERSION
+ spec.version = GitLab::License::Scanning::VERSION
spec.authors = ['Fabien Catteau', 'Olivier Gonzalez', 'mo khan']
spec.email = ['fcatteau@gitlab.com', 'ogonzalez@gitlab.com', 'mkhan@gitlab.com']
@@ -27,6 +27,7 @@ Gem::Specification.new do |spec|
spec.require_paths = ['lib']
spec.add_dependency 'license_finder', '~> 6.0.0'
+ spec.add_dependency 'spandx', '~> 0.1'
spec.add_development_dependency 'json-schema', '~> 2.8'
spec.add_development_dependency 'rspec', '~> 3.9'
end
diff --git a/lib/gitlab/license/scanning.rb b/lib/gitlab/license/scanning.rb
new file mode 100644
index 0000000..f8fa375
--- /dev/null
+++ b/lib/gitlab/license/scanning.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require 'pathname'
+require 'yaml'
+require 'json'
+require 'license_finder'
+require 'spandx'
+require 'gitlab/license/scanning/loggable'
+require 'gitlab/license/scanning/verifiable'
+require 'gitlab/license/scanning/repository'
+require 'gitlab/license/scanning/report'
+require 'gitlab/license/scanning/version'
+
+# This applies a monkey patch to the JsonReport found in the `license_finder` gem.
+LicenseFinder::JsonReport.prepend(GitLab::License::Scanning::Report)
+
+module GitLab
+ module License
+ module Scanning
+ def self.root
+ Pathname.new(File.dirname(__FILE__)).join('../../..')
+ end
+
+ def self.http
+ @http ||= Net::Hippie::Client.new.tap do |client|
+ client.logger = ::Logger.new('http.log')
+ client.follow_redirects = 3
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/loggable.rb b/lib/gitlab/license/scanning/loggable.rb
new file mode 100644
index 0000000..e101e82
--- /dev/null
+++ b/lib/gitlab/license/scanning/loggable.rb
@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ module Loggable
+ def logger
+ ::LicenseFinder::Core.default_logger
+ end
+
+ def log_info(message)
+ logger.info(self.class, message)
+ end
+
+ def log_error(message)
+ logger.info(self.class, message, color: :red)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/report.rb b/lib/gitlab/license/scanning/report.rb
new file mode 100644
index 0000000..102dea6
--- /dev/null
+++ b/lib/gitlab/license/scanning/report.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require 'gitlab/license/scanning/report/base'
+require 'gitlab/license/scanning/report/v1'
+require 'gitlab/license/scanning/report/v1_1'
+require 'gitlab/license/scanning/report/v2'
+
+module GitLab
+ module License
+ module Scanning
+ module Report
+ DEFAULT_VERSION = '1'
+ VERSIONS = {
+ nil => V1,
+ '' => V1,
+ '1' => V1,
+ '1.0' => V1,
+ '1.1' => V1_1,
+ '2' => V2,
+ '2.0' => V2
+ }.freeze
+
+ # This method overrides the method defined in `LicenseFinder::JsonReport` to
+ # allow us to generate a custom json report.
+ def to_s
+ JSON.pretty_generate(version_for(report_version).to_h) + "\n"
+ end
+
+ private
+
+ def report_version
+ ENV.fetch('LM_REPORT_VERSION', DEFAULT_VERSION)
+ end
+
+ def version_for(version)
+ VERSIONS.fetch(version.to_s).new(dependencies)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/report/base.rb b/lib/gitlab/license/scanning/report/base.rb
new file mode 100644
index 0000000..b8d0219
--- /dev/null
+++ b/lib/gitlab/license/scanning/report/base.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ module Report
+ class Base
+ include Loggable
+ include Verifiable
+
+ attr_reader :dependencies, :repository
+
+ def initialize(dependencies)
+ @dependencies = dependencies.sort
+ @repository = Repository.new
+ end
+
+ def to_h
+ raise NotImplementedError
+ end
+
+ private
+
+ def paths_from(dependency)
+ return [] unless dependency.respond_to?(:aggregate_paths)
+
+ paths = dependency.aggregate_paths
+ return [] if blank?(paths)
+
+ paths.map { |x| x.gsub(Dir.pwd, '.') }
+ end
+
+ def description_for(dependency)
+ present?(dependency.summary) ? dependency.summary : dependency.description
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/report/v1.rb b/lib/gitlab/license/scanning/report/v1.rb
new file mode 100644
index 0000000..faece8d
--- /dev/null
+++ b/lib/gitlab/license/scanning/report/v1.rb
@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ module Report
+ class V1 < Base
+ def to_h
+ {
+ licenses: license_summary,
+ dependencies: formatted_dependencies(dependencies)
+ }
+ end
+
+ private
+
+ # when a dependency has multiple licenses, this will join the licenses into a single name
+ # this defect was backported from the [html2json](https://gitlab.com/gitlab-org/security-products/license-management/blob/7f175952a5a047d785b5ea72c15a10642523c62a/html2json.js) version of this script.
+ def license_summary
+ dependencies
+ .map { |dependency| join_license_names(dependency.licenses) }
+ .flatten
+ .group_by { |name| name }
+ .map { |license, items| { count: items.count, name: license } }
+ .sort_by { |hash| [-hash[:count], hash[:name]] }
+ end
+
+ # when a dependency has more than one license
+ # this method chooses one of the urls.
+ # to maintain backwards compatibility this bug has been carried forward.
+ def license_for(dependency)
+ license = { name: join_license_names(dependency.licenses) }
+
+ urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort
+ log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1
+ url = urls[0] || license_data(dependency.licenses.first)['url']
+
+ license[:url] = url if present?(url)
+ license
+ end
+
+ def join_license_names(licenses)
+ licenses.map { |x| best_name_for(x) }.sort.reverse.join(', ')
+ end
+
+ def map_from_dependency(dependency)
+ result = {
+ license: license_for(dependency),
+ dependency: {
+ name: dependency.name,
+ url: dependency.homepage,
+ description: description_for(dependency),
+ pathes: paths_from(dependency)
+ }
+ }
+ result[:dependency].delete(:url) if blank?(dependency.homepage)
+ result
+ end
+
+ def formatted_dependencies(dependencies)
+ dependencies.map { |x| map_from_dependency(x) }
+ end
+
+ def best_name_for(license)
+ license_data(license)['name']
+ end
+
+ def license_data(license)
+ {
+ 'name' => license.name.split(/[\r\n]+/)[0],
+ 'url' => license.url || ''
+ }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/report/v1_1.rb b/lib/gitlab/license/scanning/report/v1_1.rb
new file mode 100644
index 0000000..a40f87f
--- /dev/null
+++ b/lib/gitlab/license/scanning/report/v1_1.rb
@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ module Report
+ class V1_1 < V1
+ def to_h
+ { version: '1.1' }.merge(super)
+ end
+
+ private
+
+ def map_from_dependency(dependency)
+ licenses = dependency.licenses.sort_by(&:name).map do |license|
+ item = license_data(license)
+ {
+ name: item['name'],
+ url: item.fetch('url', '')
+ }
+ end
+
+ { licenses: licenses }.merge(super)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/report/v2.rb b/lib/gitlab/license/scanning/report/v2.rb
new file mode 100644
index 0000000..db7d7f0
--- /dev/null
+++ b/lib/gitlab/license/scanning/report/v2.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ module Report
+ class V2 < Base
+ def to_h
+ {
+ version: '2.0',
+ licenses: license_summary,
+ dependencies: dependencies.map { |x| map_from(x) }
+ }
+ end
+
+ private
+
+ def all_licenses
+ dependencies.map { |x| x.licenses.to_a }.flatten
+ end
+
+ def license_summary
+ all_licenses
+ .group_by { |x| data_for(x)['name'] }
+ .sort_by { |x, y| [-y.size, x] }
+ .map { |_name, items| data_for(items[0]).merge(count: items.count) }
+ end
+
+ def data_for(license)
+ repository.item_for(license)
+ end
+
+ def map_from(dependency)
+ licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort
+ log_info [dependency.name, dependency.version, licenses].inspect
+
+ {
+ name: dependency.name,
+ url: dependency.homepage,
+ description: description_for(dependency),
+ paths: paths_from(dependency),
+ licenses: licenses
+ }
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/repository.rb b/lib/gitlab/license/scanning/repository.rb
new file mode 100644
index 0000000..13b975a
--- /dev/null
+++ b/lib/gitlab/license/scanning/repository.rb
@@ -0,0 +1,95 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ class Repository
+ include Loggable
+ include Verifiable
+ KNOWN_SOURCES = [
+ 'licenses.nuget.org',
+ 'opensource.org',
+ 'www.opensource.org',
+ ].freeze
+
+ def initialize(compatibility_path: GitLab::License::Scanning.root.join('normalized-licenses.yml'))
+ @compatibility_data = YAML.safe_load(IO.read(compatibility_path))
+ @catalogue = Spandx::Catalogue.latest
+ end
+
+ def item_for(license)
+ spdx_data_for(id_for(license)) ||
+ spdx_data_for(license.send(:short_name)) ||
+ generate_item_for(license)
+ end
+
+ private
+
+ attr_reader :catalogue, :compatibility_data
+
+ def spdx_data_for(id)
+ return if blank?(id)
+
+ data = catalogue[id]
+ if data
+ {
+ 'id' => data.id,
+ 'name' => data.name,
+ 'url' => data.see_also[-1]
+ }
+ end
+ end
+
+ def id_for(license)
+ ids = compatibility_data['ids']
+ ids[license.send(:short_name)] ||
+ ids[license.url] ||
+ known_sources(license.send(:short_name)) ||
+ known_sources(license.url)
+ end
+
+ # When `license_finder` is unable to determine the license it will use the full
+ # content of the file as the name of the license. This method shrinks that name
+ # down to just take the first line of the file.
+ def take_first_line_from(content)
+ return '' if blank?(content)
+
+ content.split(/[\r\n]+/)[0]
+ end
+
+ def generate_item_for(license)
+ log_info("Detected unknown license `#{license.send(:short_name)}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.")
+ name = take_first_line_from(license.name)
+ {
+ 'id' => name.downcase,
+ 'name' => name,
+ 'url' => present?(license.url) ? license.url : ''
+ }
+ end
+
+ def load_spdx_data_from(path)
+ content = IO.read(path)
+ json = JSON.parse(content)
+ licenses = json['licenses']
+
+ licenses.inject({}) do |memo, license|
+ memo[license['licenseId']] = license
+ memo
+ end
+ end
+
+ def known_sources(url)
+ return if blank?(url)
+ return unless url =~ /\A#{::URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/
+
+ uri = URI.parse(url)
+ return unless KNOWN_SOURCES.include?(uri.host.downcase)
+ uri.path.split('/')[-1]
+ rescue => error
+ log_info(error)
+ nil
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/verifiable.rb b/lib/gitlab/license/scanning/verifiable.rb
new file mode 100644
index 0000000..1d24c53
--- /dev/null
+++ b/lib/gitlab/license/scanning/verifiable.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ module Verifiable
+ def blank?(item)
+ item.nil? || item.empty?
+ end
+
+ def present?(item)
+ !blank?(item)
+ end
+ end
+ end
+ end
+end
diff --git a/lib/gitlab/license/scanning/version.rb b/lib/gitlab/license/scanning/version.rb
new file mode 100644
index 0000000..545efc1
--- /dev/null
+++ b/lib/gitlab/license/scanning/version.rb
@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module GitLab
+ module License
+ module Scanning
+ VERSION = '2.7.0'
+ end
+ end
+end
diff --git a/lib/license/management.rb b/lib/license/management.rb
deleted file mode 100644
index 37fe1a0..0000000
--- a/lib/license/management.rb
+++ /dev/null
@@ -1,29 +0,0 @@
-# frozen_string_literal: true
-
-require 'pathname'
-require 'yaml'
-require 'json'
-require 'license_finder'
-require 'license/management/loggable'
-require 'license/management/verifiable'
-require 'license/management/repository'
-require 'license/management/report'
-require 'license/management/version'
-
-# This applies a monkey patch to the JsonReport found in the `license_finder` gem.
-LicenseFinder::JsonReport.prepend(License::Management::Report)
-
-module License
- module Management
- def self.root
- Pathname.new(File.dirname(__FILE__)).join('../..')
- end
-
- def self.http
- @http ||= Net::Hippie::Client.new.tap do |client|
- client.logger = ::Logger.new('http.log')
- client.follow_redirects = 3
- end
- end
- end
-end
diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb
deleted file mode 100644
index 165d8ca..0000000
--- a/lib/license/management/loggable.rb
+++ /dev/null
@@ -1,19 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- module Loggable
- def logger
- ::LicenseFinder::Core.default_logger
- end
-
- def log_info(message)
- logger.info(self.class, message)
- end
-
- def log_error(message)
- logger.info(self.class, message, color: :red)
- end
- end
- end
-end
diff --git a/lib/license/management/report.rb b/lib/license/management/report.rb
deleted file mode 100644
index aa3111d..0000000
--- a/lib/license/management/report.rb
+++ /dev/null
@@ -1,39 +0,0 @@
-# frozen_string_literal: true
-
-require 'license/management/report/base'
-require 'license/management/report/v1'
-require 'license/management/report/v1_1'
-require 'license/management/report/v2'
-
-module License
- module Management
- module Report
- DEFAULT_VERSION = '1'
- VERSIONS = {
- nil => V1,
- '' => V1,
- '1' => V1,
- '1.0' => V1,
- '1.1' => V1_1,
- '2' => V2,
- '2.0' => V2
- }.freeze
-
- # This method overrides the method defined in `LicenseFinder::JsonReport` to
- # allow us to generate a custom json report.
- def to_s
- JSON.pretty_generate(version_for(report_version).to_h) + "\n"
- end
-
- private
-
- def report_version
- ENV.fetch('LM_REPORT_VERSION', DEFAULT_VERSION)
- end
-
- def version_for(version)
- VERSIONS.fetch(version.to_s).new(dependencies)
- end
- end
- end
-end
diff --git a/lib/license/management/report/base.rb b/lib/license/management/report/base.rb
deleted file mode 100644
index 2d49341..0000000
--- a/lib/license/management/report/base.rb
+++ /dev/null
@@ -1,38 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- module Report
- class Base
- include Loggable
- include Verifiable
-
- attr_reader :dependencies, :repository
-
- def initialize(dependencies)
- @dependencies = dependencies.sort
- @repository = License::Management::Repository.new
- end
-
- def to_h
- raise NotImplementedError
- end
-
- private
-
- def paths_from(dependency)
- return [] unless dependency.respond_to?(:aggregate_paths)
-
- paths = dependency.aggregate_paths
- return [] if blank?(paths)
-
- paths.map { |x| x.gsub(Dir.pwd, '.') }
- end
-
- def description_for(dependency)
- present?(dependency.summary) ? dependency.summary : dependency.description
- end
- end
- end
- end
-end
diff --git a/lib/license/management/report/v1.rb b/lib/license/management/report/v1.rb
deleted file mode 100644
index 49423c6..0000000
--- a/lib/license/management/report/v1.rb
+++ /dev/null
@@ -1,76 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- module Report
- class V1 < Base
- def to_h
- {
- licenses: license_summary,
- dependencies: formatted_dependencies(dependencies)
- }
- end
-
- private
-
- # when a dependency has multiple licenses, this will join the licenses into a single name
- # this defect was backported from the [html2json](https://gitlab.com/gitlab-org/security-products/license-management/blob/7f175952a5a047d785b5ea72c15a10642523c62a/html2json.js) version of this script.
- def license_summary
- dependencies
- .map { |dependency| join_license_names(dependency.licenses) }
- .flatten
- .group_by { |name| name }
- .map { |license, items| { count: items.count, name: license } }
- .sort_by { |hash| [-hash[:count], hash[:name]] }
- end
-
- # when a dependency has more than one license
- # this method chooses one of the urls.
- # to maintain backwards compatibility this bug has been carried forward.
- def license_for(dependency)
- license = { name: join_license_names(dependency.licenses) }
-
- urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort
- log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1
- url = urls[0] || license_data(dependency.licenses.first)['url']
-
- license[:url] = url if present?(url)
- license
- end
-
- def join_license_names(licenses)
- licenses.map { |x| best_name_for(x) }.sort.reverse.join(', ')
- end
-
- def map_from_dependency(dependency)
- result = {
- license: license_for(dependency),
- dependency: {
- name: dependency.name,
- url: dependency.homepage,
- description: description_for(dependency),
- pathes: paths_from(dependency)
- }
- }
- result[:dependency].delete(:url) if blank?(dependency.homepage)
- result
- end
-
- def formatted_dependencies(dependencies)
- dependencies.map { |x| map_from_dependency(x) }
- end
-
- def best_name_for(license)
- license_data(license)['name']
- end
-
- def license_data(license)
- {
- 'name' => license.name.split(/[\r\n]+/)[0],
- 'url' => license.url || ''
- }
- end
- end
- end
- end
-end
diff --git a/lib/license/management/report/v1_1.rb b/lib/license/management/report/v1_1.rb
deleted file mode 100644
index 1186e2a..0000000
--- a/lib/license/management/report/v1_1.rb
+++ /dev/null
@@ -1,27 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- module Report
- class V1_1 < V1
- def to_h
- { version: '1.1' }.merge(super)
- end
-
- private
-
- def map_from_dependency(dependency)
- licenses = dependency.licenses.sort_by(&:name).map do |license|
- item = license_data(license)
- {
- name: item['name'],
- url: item.fetch('url', '')
- }
- end
-
- { licenses: licenses }.merge(super)
- end
- end
- end
- end
-end
diff --git a/lib/license/management/report/v2.rb b/lib/license/management/report/v2.rb
deleted file mode 100644
index 6ab6b99..0000000
--- a/lib/license/management/report/v2.rb
+++ /dev/null
@@ -1,47 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- module Report
- class V2 < Base
- def to_h
- {
- version: '2.0',
- licenses: license_summary,
- dependencies: dependencies.map { |x| map_from(x) }
- }
- end
-
- private
-
- def all_licenses
- dependencies.map { |x| x.licenses.to_a }.flatten
- end
-
- def license_summary
- all_licenses
- .group_by { |x| data_for(x)['name'] }
- .sort_by { |x, y| [-y.size, x] }
- .map { |_name, items| data_for(items[0]).merge(count: items.count) }
- end
-
- def data_for(license)
- repository.item_for(license)
- end
-
- def map_from(dependency)
- licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort
- log_info [dependency.name, dependency.version, licenses].inspect
-
- {
- name: dependency.name,
- url: dependency.homepage,
- description: description_for(dependency),
- paths: paths_from(dependency),
- licenses: licenses
- }
- end
- end
- end
- end
-end
diff --git a/lib/license/management/repository.rb b/lib/license/management/repository.rb
deleted file mode 100644
index 9490af2..0000000
--- a/lib/license/management/repository.rb
+++ /dev/null
@@ -1,96 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- class Repository
- include Loggable
- include Verifiable
- KNOWN_SOURCES = [
- 'licenses.nuget.org',
- 'opensource.org',
- 'www.opensource.org',
- ].freeze
-
- def initialize(
- compatibility_path: License::Management.root.join('normalized-licenses.yml'),
- spdx_path: License::Management.root.join('spdx-licenses.json')
- )
- @compatibility_data = YAML.safe_load(IO.read(compatibility_path))
- @spdx_data = load_spdx_data_from(spdx_path)
- end
-
- def item_for(license)
- spdx_data_for(id_for(license)) ||
- spdx_data_for(license.send(:short_name)) ||
- generate_item_for(license)
- end
-
- private
-
- attr_reader :spdx_data, :compatibility_data
-
- def spdx_data_for(id)
- return if blank?(id)
-
- data = spdx_data[id]
- if data
- {
- 'id' => data['licenseId'],
- 'name' => data['name'],
- 'url' => data['seeAlso'][-1]
- }
- end
- end
-
- def id_for(license)
- ids = compatibility_data['ids']
- ids[license.send(:short_name)] ||
- ids[license.url] ||
- known_sources(license.send(:short_name)) ||
- known_sources(license.url)
- end
-
- # When `license_finder` is unable to determine the license it will use the full
- # content of the file as the name of the license. This method shrinks that name
- # down to just take the first line of the file.
- def take_first_line_from(content)
- return '' if blank?(content)
-
- content.split(/[\r\n]+/)[0]
- end
-
- def generate_item_for(license)
- log_info("Detected unknown license `#{license.send(:short_name)}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.")
- name = take_first_line_from(license.name)
- {
- 'id' => name.downcase,
- 'name' => name,
- 'url' => present?(license.url) ? license.url : ''
- }
- end
-
- def load_spdx_data_from(path)
- content = IO.read(path)
- json = JSON.parse(content)
- licenses = json['licenses']
-
- licenses.inject({}) do |memo, license|
- memo[license['licenseId']] = license
- memo
- end
- end
-
- def known_sources(url)
- return if blank?(url)
- return unless url =~ /\A#{::URI::DEFAULT_PARSER.make_regexp(['http', 'https'])}\z/
-
- uri = URI.parse(url)
- return unless KNOWN_SOURCES.include?(uri.host.downcase)
- uri.path.split('/')[-1]
- rescue => error
- log_info(error)
- nil
- end
- end
- end
-end
diff --git a/lib/license/management/verifiable.rb b/lib/license/management/verifiable.rb
deleted file mode 100644
index 72667fb..0000000
--- a/lib/license/management/verifiable.rb
+++ /dev/null
@@ -1,15 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- module Verifiable
- def blank?(item)
- item.nil? || item.empty?
- end
-
- def present?(item)
- !blank?(item)
- end
- end
- end
-end
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
deleted file mode 100644
index 7a3cc03..0000000
--- a/lib/license/management/version.rb
+++ /dev/null
@@ -1,7 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- VERSION = '2.7.0'
- end
-end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 15c9dad..1e48a8f 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -1,4 +1,4 @@
-require 'license/management'
+require 'gitlab/license/scanning'
require 'json'
require 'securerandom'
require 'json-schema'
diff --git a/spec/unit/license/management/report/v2_spec.rb b/spec/unit/report/v2_spec.rb
index 4da973c..7bbbc3d 100644
--- a/spec/unit/license/management/report/v2_spec.rb
+++ b/spec/unit/report/v2_spec.rb
@@ -1,11 +1,10 @@
-RSpec.describe License::Management::Report::V2 do
+RSpec.describe GitLab::License::Scanning::Report::V2 do
describe "#to_h" do
{
'AGPL-1.0' => 'AGPL-1.0',
'AGPL-3.0' => 'AGPL-3.0',
'Apache 2.0' => 'Apache-2.0',
'Artistic-2.0' => 'Artistic-2.0',
- 'BSD' => 'BSD-4-Clause',
'CC0 1.0 Universal' => 'CC0-1.0',
'CDDL-1.0' => 'CDDL-1.0',
'CDDL-1.1' => 'CDDL-1.1',
diff --git a/spec/unit/license/management/repository_spec.rb b/spec/unit/repository_spec.rb
index 6ebc09e..fa00297 100644
--- a/spec/unit/license/management/repository_spec.rb
+++ b/spec/unit/repository_spec.rb
@@ -1,11 +1,11 @@
-RSpec.describe License::Management::Repository do
+RSpec.describe GitLab::License::Scanning::Repository do
describe "#item_for" do
- let(:spdx_licenses) { JSON.parse(IO.read('spdx-licenses.json'))['licenses'] }
+ let(:spdx_licenses) { Spandx::Catalogue.latest }
context "when mapping a license that refers to opensource.org" do
it 'parses the SPDX id from the url' do
spdx_licenses.each do |license|
- spdx_id = license['licenseId']
+ spdx_id = license.id
url = "https://opensource.org/licenses/#{spdx_id}"
license = LicenseFinder::License.new(short_name: url, matcher: LicenseFinder::License::NoneMatcher.new, url: url)
expect(subject.item_for(license)['id']).to eql(spdx_id)
@@ -16,7 +16,7 @@ RSpec.describe License::Management::Repository do
context "when mapping a license that refers to nuget.org" do
it 'parses the SPDX id from the url' do
spdx_licenses.each do |license|
- spdx_id = license['licenseId']
+ spdx_id = license.id
url = "https://licenses.nuget.org/#{spdx_id}"
license = LicenseFinder::License.new(short_name: url, matcher: LicenseFinder::License::NoneMatcher.new, url: url)
expect(subject.item_for(license)['id']).to eql(spdx_id)