From 6001e7586f04528067b16c08a6b046be5b5b62ec Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 3 Nov 2020 15:38:05 -0700 Subject: use recursive scan by default docs: update CHANGELOG and version feat: scan packages in parallel fix: prevent infinite recursion --- lib/license/finder/ext.rb | 3 +++ lib/license/finder/ext/bower.rb | 4 ++-- lib/license/finder/ext/conan.rb | 4 ++-- lib/license/finder/ext/go_dep.rb | 19 +++++++++++++++++++ lib/license/finder/ext/go_modules.rb | 8 +++++++- lib/license/finder/ext/npm.rb | 6 ++++-- lib/license/finder/ext/scanner.rb | 13 +++++++++++++ lib/license/finder/ext/trash.rb | 19 +++++++++++++++++++ lib/license/finder/ext/yarn.rb | 6 ++++-- lib/license/management/report/base.rb | 2 -- lib/license/management/report/v2.rb | 5 ----- lib/license/management/version.rb | 2 +- 12 files changed, 74 insertions(+), 17 deletions(-) create mode 100644 lib/license/finder/ext/go_dep.rb create mode 100644 lib/license/finder/ext/scanner.rb create mode 100644 lib/license/finder/ext/trash.rb (limited to 'lib/license') diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb index fb593cc..3780fb2 100644 --- a/lib/license/finder/ext.rb +++ b/lib/license/finder/ext.rb @@ -7,6 +7,7 @@ require 'license/finder/ext/composer' require 'license/finder/ext/conan' require 'license/finder/ext/dependency' require 'license/finder/ext/dotnet' +require 'license/finder/ext/go_dep' require 'license/finder/ext/go_modules' require 'license/finder/ext/gradle' require 'license/finder/ext/license' @@ -16,7 +17,9 @@ require 'license/finder/ext/nuget' require 'license/finder/ext/package_manager' require 'license/finder/ext/pip' require 'license/finder/ext/pipenv' +require 'license/finder/ext/scanner' require 'license/finder/ext/shared_helpers' +require 'license/finder/ext/trash' require 'license/finder/ext/yarn' # Apply patch to the JsonReport found in the `license_finder` gem. diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb index 52e6a16..2725e2f 100644 --- a/lib/license/finder/ext/bower.rb +++ b/lib/license/finder/ext/bower.rb @@ -2,8 +2,8 @@ module LicenseFinder class Bower - def possible_package_paths - [project_path.join('bower.json')] + def active? + project_path.join('bower.json').exist? end def prepare diff --git a/lib/license/finder/ext/conan.rb b/lib/license/finder/ext/conan.rb index 90e7d9b..26e45aa 100644 --- a/lib/license/finder/ext/conan.rb +++ b/lib/license/finder/ext/conan.rb @@ -2,8 +2,8 @@ module LicenseFinder class Conan - def possible_package_paths - [project_path.join('conanfile.txt')] + def active? + project_path.join('conanfile.txt').exist? end def prepare diff --git a/lib/license/finder/ext/go_dep.rb b/lib/license/finder/ext/go_dep.rb new file mode 100644 index 0000000..f2867d3 --- /dev/null +++ b/lib/license/finder/ext/go_dep.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module LicenseFinder + class GoDep + def active? + return if project_path.to_path.include?('/vendor/') + + project_path.join('Godeps/Godeps.json').exist? + end + + def prepare + within_project_path do + tool_box.install(tool: :golang) + shell.execute([:go, :install, '-i', 'github.com/golang/dep/cmd/dep'], capture: false) + shell.execute([:asdf, :reshim], capture: false) + end + end + end +end diff --git a/lib/license/finder/ext/go_modules.rb b/lib/license/finder/ext/go_modules.rb index 8927f2c..8a9ea03 100644 --- a/lib/license/finder/ext/go_modules.rb +++ b/lib/license/finder/ext/go_modules.rb @@ -5,6 +5,12 @@ module LicenseFinder FORMAT = "'{{.Main}},{{.Path}},{{.Version}},{{.Dir}}'" HEADER = [:main_module, :name, :version, :dir].freeze + def active? + return if project_path.to_path.include?('/vendor/') + + go_sum_path.exist? + end + def prepare return if vendored? @@ -59,7 +65,7 @@ module LicenseFinder end def go_sum_path - @go_sum_path ||= Pathname.glob(project_path.join('go.sum')).find(&:exist?) + @go_sum_path ||= project_path.join('go.sum') end def vendor_path diff --git a/lib/license/finder/ext/npm.rb b/lib/license/finder/ext/npm.rb index 59244c9..59ae24c 100644 --- a/lib/license/finder/ext/npm.rb +++ b/lib/license/finder/ext/npm.rb @@ -2,8 +2,10 @@ module LicenseFinder class NPM - def possible_package_paths - [project_path.join('package.json')] + def active? + return if project_path.to_path.include?('/node_modules/') + + project_path.join('package.json').exist? end def prepare diff --git a/lib/license/finder/ext/scanner.rb b/lib/license/finder/ext/scanner.rb new file mode 100644 index 0000000..1c5e4ae --- /dev/null +++ b/lib/license/finder/ext/scanner.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module LicenseFinder + class Scanner + def active_packages + active_package_managers + .select { |x| x.installed?(@logger) } + .map { |x| Thread.new { x.current_packages_with_relations } } + .map(&:value) + .flatten + end + end +end diff --git a/lib/license/finder/ext/trash.rb b/lib/license/finder/ext/trash.rb new file mode 100644 index 0000000..76f16b1 --- /dev/null +++ b/lib/license/finder/ext/trash.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module LicenseFinder + class Trash + def active? + return if project_path.to_path.include?('/vendor/') + + project_path.join('vendor.conf').exist? + end + + def prepare + within_project_path do + tool_box.install(tool: :golang) + shell.execute([:go, :get, '-u', 'github.com/rancher/trash'], capture: false) + shell.execute([:asdf, :reshim], capture: false) + end + end + end +end diff --git a/lib/license/finder/ext/yarn.rb b/lib/license/finder/ext/yarn.rb index 7a18e35..eedf435 100644 --- a/lib/license/finder/ext/yarn.rb +++ b/lib/license/finder/ext/yarn.rb @@ -5,8 +5,10 @@ module LicenseFinder INCOMPATIBLE_PACKAGE_REGEX = /(?[\w,\-]+)@(?(\d+\.?)+)/.freeze PHANTOM_PACKAGE_REGEX = /workspace-aggregator-[a-zA-Z0-9]{8}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{4}-[a-zA-Z0-9]{12}/.freeze - def possible_package_paths - [project_path.join('yarn.lock')] + def active? + return if project_path.to_path.include?('/node_modules/') + + project_path.join('yarn.lock').exist? end def prepare diff --git a/lib/license/management/report/base.rb b/lib/license/management/report/base.rb index c2a38c7..0155c15 100644 --- a/lib/license/management/report/base.rb +++ b/lib/license/management/report/base.rb @@ -7,8 +7,6 @@ module License include Loggable include Verifiable - CONTRIBUTION_URL = "https://gitlab.com/gitlab-org/security-products/analyzers/license-finder#contributing" - attr_reader :dependencies, :repository def initialize(dependencies) diff --git a/lib/license/management/report/v2.rb b/lib/license/management/report/v2.rb index ac43f53..3cbfbab 100644 --- a/lib/license/management/report/v2.rb +++ b/lib/license/management/report/v2.rb @@ -47,11 +47,6 @@ module License def log(dependency, licenses) logger.info { [dependency.name, dependency.version, licenses].flatten.join(' ') } - return unless licenses == ['unknown'] - - logger.warn do - "Contribute #{dependency.name} #{dependency.version} to #{CONTRIBUTION_URL}" - end end end end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index 7bd04dd..6307742 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.28.3' + VERSION = '3.29.0' end end -- cgit v1.2.3