diff options
| author | mo khan <mo.khan@gmail.com> | 2020-07-13 09:57:29 -0600 |
|---|---|---|
| committer | mo khan <mo.khan@gmail.com> | 2020-08-21 16:31:57 -0600 |
| commit | cbc4d1e328fd0e578f038d57fbd6b9354dafa79e (patch) | |
| tree | 99b53a90cd7c0996a2d0993b41eb56a695bcc996 /lib | |
| parent | eb11eeb1018c4d32e8dbaf3f734e588205f0671f (diff) | |
Install tools from .deb package
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/license/finder/ext/bower.rb | 45 | ||||
| -rw-r--r-- | lib/license/finder/ext/bundler.rb | 41 | ||||
| -rw-r--r-- | lib/license/finder/ext/cargo.rb | 6 | ||||
| -rw-r--r-- | lib/license/finder/ext/composer.rb | 1 | ||||
| -rw-r--r-- | lib/license/finder/ext/conan.rb | 10 | ||||
| -rw-r--r-- | lib/license/finder/ext/dotnet.rb | 14 | ||||
| -rw-r--r-- | lib/license/finder/ext/go_modules.rb | 5 | ||||
| -rw-r--r-- | lib/license/finder/ext/gradle.rb | 25 | ||||
| -rw-r--r-- | lib/license/finder/ext/maven.rb | 29 | ||||
| -rw-r--r-- | lib/license/finder/ext/npm.rb | 34 | ||||
| -rw-r--r-- | lib/license/finder/ext/nuget.rb | 16 | ||||
| -rw-r--r-- | lib/license/finder/ext/package_manager.rb | 19 | ||||
| -rw-r--r-- | lib/license/finder/ext/pip.rb | 90 | ||||
| -rw-r--r-- | lib/license/finder/ext/pipenv.rb | 53 | ||||
| -rw-r--r-- | lib/license/finder/ext/yarn.rb | 31 | ||||
| -rw-r--r-- | lib/license/management.rb | 2 | ||||
| -rw-r--r-- | lib/license/management/python.rb | 61 | ||||
| -rw-r--r-- | lib/license/management/shell.rb | 52 | ||||
| -rw-r--r-- | lib/license/management/tool_box.rb | 123 | ||||
| -rw-r--r-- | lib/license/management/version.rb | 2 |
20 files changed, 445 insertions, 214 deletions
diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb index 9b302f6..c034568 100644 --- a/lib/license/finder/ext/bower.rb +++ b/lib/license/finder/ext/bower.rb @@ -1,33 +1,38 @@ # frozen_string_literal: true module LicenseFinder - class Bower < PackageManager - def prepare - shell.execute([ - :bower, - :install, - '--allow-root', - '--production', - '--verbose', - '--loglevel', - :debug - ], env: default_env) + class Bower + def possible_package_paths + [project_path.join('bower.json')] end - def current_packages - map_all(bower_output).flatten.compact + def prepare + within_project_path do + tool_box.install(tool: :nodejs, env: default_env) + shell.execute([ + :bower, + :install, + '--allow-root', + '--production', + '--verbose', + '--loglevel', + :debug + ], env: default_env) + end end - def possible_package_paths - [project_path.join('bower.json')] + def current_packages + within_project_path do + map_all(bower_output).flatten.compact + end end private def bower_output - stdout, _stderr, status = Dir.chdir(project_path) do - shell.execute([:bower, :list, '--json', '-l', 'action', '--allow-root'], env: default_env) - end + stdout, _stderr, status = shell.execute([ + :bower, :list, '--json', '-l', 'action', '--allow-root' + ], env: default_env) return {} unless status.success? JSON.parse(stdout) @@ -62,9 +67,5 @@ module LicenseFinder 'bower_directory' => ENV.fetch('bower_directory', vendor_path.join('bower_components')).to_s } end - - def vendor_path - Pathname.pwd.join('.gitlab', 'cache', 'vendor') - end end end diff --git a/lib/license/finder/ext/bundler.rb b/lib/license/finder/ext/bundler.rb index 0530f32..b8c755b 100644 --- a/lib/license/finder/ext/bundler.rb +++ b/lib/license/finder/ext/bundler.rb @@ -1,14 +1,25 @@ # frozen_string_literal: true module LicenseFinder - class Bundler < PackageManager + class Bundler def prepare create_vendor_path with_clean_bundler do - _stdout, _stderr, status = shell.execute([:asdf, :current, :ruby], env: default_env) - shell.execute([:asdf, :install], env: default_env) unless status.success? - shell.execute([:bundle, :config, '--local', :path, vendor_path.to_s], env: default_env) + tool_box.install(tool: :ruby, version: ruby_version, env: default_env) + Pathname.glob('/opt/toolcache/*.gem').each do |gem| + shell.execute([ + :gem, :install, gem, + '--no-document', + '--verbose', + '--no-update-sources', + '--ignore-dependencies', + '--no-suggestions', + '--local' + ], env: default_env) + end + shell.execute([:asdf, :reshim], env: default_env) + shell.execute([:bundle, :config, '--local', :path, vendor_path], env: default_env) shell.execute([:bundle, :install, '--verbose'], env: default_env) end end @@ -53,11 +64,11 @@ module LicenseFinder def scan_command [ - '/opt/asdf/shims/ruby', + :ruby, '-W0', - ::License::Management.root.join('exe', 'scan_bundler').to_s, - detected_package_path.to_s, - lockfile_path.to_s + ::License::Management.root.join('exe', 'scan_bundler'), + detected_package_path, + lockfile_path ] end @@ -91,6 +102,10 @@ module LicenseFinder end end + def ruby_version + @ruby_version ||= gemfile_ruby_version || tool_box.version_of(:ruby) + end + def map_from(gem) Dependency.new( 'Bundler', @@ -104,5 +119,15 @@ module LicenseFinder summary: gem[:summary] ) end + + def gemfile_ruby_version + with_clean_bundler do + stdout, _stderr, status = shell.execute([:embedded_bundle, :platform, '--ruby']) + return unless status.success? + + match = stdout.chomp.match(/\Aruby (?<version>\d+\.\d+.\d+).*\z/) + match[:version] if match + end + end end end diff --git a/lib/license/finder/ext/cargo.rb b/lib/license/finder/ext/cargo.rb index 20b1b03..c957e0e 100644 --- a/lib/license/finder/ext/cargo.rb +++ b/lib/license/finder/ext/cargo.rb @@ -12,9 +12,8 @@ module LicenseFinder create_vendor_path within_project_path do - shell.execute([:dpkg, '-i', '/opt/toolcache/rust*.deb']) - shell.execute([:asdf, :reshim]) - shell.execute([:env, '|', :sort, '&&', :asdf, :exec, :cargo, :fetch, '-vv'], env: default_env) + tool_box.install(tool: :rust) + shell.execute([:cargo, :fetch, '-vv'], env: default_env) end end @@ -44,7 +43,6 @@ module LicenseFinder def scan_command [ - :asdf, :exec, :cargo, :tree, '--edges=no-dev', "--color=never", "--offline", diff --git a/lib/license/finder/ext/composer.rb b/lib/license/finder/ext/composer.rb index 1434798..993119c 100644 --- a/lib/license/finder/ext/composer.rb +++ b/lib/license/finder/ext/composer.rb @@ -6,6 +6,7 @@ module LicenseFinder create_vendor_path within_project_path do + tool_box.install(tool: :php) shell.execute([ :composer, :install, diff --git a/lib/license/finder/ext/conan.rb b/lib/license/finder/ext/conan.rb index ca2ee03..780b205 100644 --- a/lib/license/finder/ext/conan.rb +++ b/lib/license/finder/ext/conan.rb @@ -6,10 +6,16 @@ module LicenseFinder [project_path.join('conanfile.txt')] end - def current_packages - stdout, _stderr, status = Dir.chdir(project_path) do + def prepare + within_project_path do + tool_box.install(tool: :python) shell.execute([:conan, :install, '--build=missing', '.'], env: default_env) shell.execute([:conan, :inspect, '.'], env: default_env) + end + end + + def current_packages + stdout, _stderr, status = within_project_path do shell.execute([:conan, :info, '-j', '/dev/stdout', '.'], env: default_env) end return [] unless status.success? diff --git a/lib/license/finder/ext/dotnet.rb b/lib/license/finder/ext/dotnet.rb index 1a7eedb..eebbbbd 100644 --- a/lib/license/finder/ext/dotnet.rb +++ b/lib/license/finder/ext/dotnet.rb @@ -10,17 +10,19 @@ module LicenseFinder end def installed?(*) - File.exist?('/opt/asdf/installs/dotnet/latest/dotnet') + true end def prepare create_vendor_path + tool_box.install(tool: :"dotnet-core", version: dotnet_version) + shell.execute([ - '/opt/asdf/installs/dotnet/latest/dotnet', - :restore, detected_package_path.to_s, + "/opt/asdf/installs/dotnet-core/#{dotnet_version}/dotnet", + :restore, detected_package_path, '--locked-mode', '--no-cache', - '--packages', vendor_path.to_s, + '--packages', vendor_path, '--verbosity', :normal ]) end @@ -35,6 +37,10 @@ module LicenseFinder end end + def dotnet_version + @dotnet_version ||= tool_box.version_of(:"dotnet-core") + end + private def map_from(name, version, data) diff --git a/lib/license/finder/ext/go_modules.rb b/lib/license/finder/ext/go_modules.rb index d22c59c..63b4772 100644 --- a/lib/license/finder/ext/go_modules.rb +++ b/lib/license/finder/ext/go_modules.rb @@ -8,7 +8,10 @@ module LicenseFinder def prepare return if vendored? - shell.execute([:go, :mod, :download, '-json']) + within_project_path do + tool_box.install(tool: :golang) + shell.execute([:go, :mod, :download, '-json']) + end end def current_packages diff --git a/lib/license/finder/ext/gradle.rb b/lib/license/finder/ext/gradle.rb index 8328300..4d6b000 100644 --- a/lib/license/finder/ext/gradle.rb +++ b/lib/license/finder/ext/gradle.rb @@ -2,6 +2,12 @@ module LicenseFinder class Gradle + def prepare + within_project_path do + tool_box.install(tool: :java, version: java_version, env: default_env) + end + end + def current_packages return [] unless download_licenses @@ -17,13 +23,21 @@ module LicenseFinder private + def java_version(env: ENV) + @java_version ||= tool_box.version_of(:java, env: env) + end + def download_licenses - _stdout, _stderr, status = Dir.chdir(project_path) do + _stdout, _stderr, status = within_project_path do + env = { + 'JAVA_HOME' => ENV.fetch("JAVA_HOME", "/opt/asdf/installs/java/#{java_version}"), + 'TERM' => 'noop' + } shell.execute([ @command, ENV.fetch('GRADLE_CLI_OPTS', '--exclude-task=test --no-daemon --debug'), 'downloadLicenses' - ], env: { 'TERM' => 'noop' }) + ], env: env) end status.success? @@ -46,5 +60,12 @@ module LicenseFinder def map_from(hash) Dependency.from(GradlePackage.new(hash, include_groups: @include_groups), detected_package_path) end + + def default_env + @default_env = { + 'CACHE_DIR' => '/opt/gitlab', + 'JAVA_HOME' => ENV.fetch("JAVA_HOME", "/opt/asdf/installs/java/#{java_version}") + } + end end end diff --git a/lib/license/finder/ext/maven.rb b/lib/license/finder/ext/maven.rb index 6c95b1d..ae8c9a2 100644 --- a/lib/license/finder/ext/maven.rb +++ b/lib/license/finder/ext/maven.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true module LicenseFinder - Maven.class_eval do + class Maven XML_PARSE_OPTIONS = { 'ForceArray' => %w[license dependency], 'GroupTags' => { @@ -10,10 +10,15 @@ module LicenseFinder } }.freeze + def prepare + within_project_path do + tool_box.install(tool: :java, version: java_version, env: default_env) + end + end + def current_packages - Dir.chdir(project_path) do - _stdout, _stderr, status = shell.execute(detect_licenses_command) - return [] unless status.success? + within_project_path do + return [] unless shell.execute(detect_licenses_command, env: default_env)[-1].success? resource_files.flat_map { |file| map_from(file.read) }.uniq end @@ -21,9 +26,21 @@ module LicenseFinder private + def java_version(env: ENV) + @java_version ||= tool_box.version_of(:java, env: env) + end + + def default_env + @default_env = { + 'CACHE_DIR' => '/opt/gitlab', + 'JAVA_HOME' => ENV.fetch("JAVA_HOME", "/opt/asdf/installs/java/#{java_version}") + } + end + def detect_licenses_command + mvn_wrapper = project_path.join('mvnw') [ - package_management_command, + mvn_wrapper.exist? ? mvn_wrapper : :mvn, "-e", "org.codehaus.mojo:license-maven-plugin:aggregate-download-licenses", "-Dlicense.excludedScopes=#{@ignored_groups.to_a.join(',')}", @@ -37,7 +54,7 @@ module LicenseFinder end def map_from(xml) - ::License::Management.logger.debug(xml) + log.debug(xml) XmlSimple .xml_in(xml, XML_PARSE_OPTIONS)['dependencies'] .map { |dependency| Dependency.from(MavenPackage.new(dependency), detected_package_path) } diff --git a/lib/license/finder/ext/npm.rb b/lib/license/finder/ext/npm.rb index e0d4e41..e245d72 100644 --- a/lib/license/finder/ext/npm.rb +++ b/lib/license/finder/ext/npm.rb @@ -2,23 +2,25 @@ module LicenseFinder class NPM - def current_packages - NpmPackage.packages_from_json(npm_json, detected_package_path).map do |item| - Dependency.from(item, detected_package_path) - end - end - - def prepare_command - lockfile? ? 'npm ci' : 'npm install --no-save' - end - def possible_package_paths [project_path.join('package.json')] end def prepare - Dir.chdir(project_path) do - shell.execute("#{prepare_command} --production", env: default_env) + within_project_path do + tool_box.install(tool: :nodejs, env: default_env) + + if lockfile? + shell.execute([:npm, :ci, "--production"], env: default_env) + else + shell.execute([:npm, :install, '--no-save', "--production"], env: default_env) + end + end + end + + def current_packages + NpmPackage.packages_from_json(npm_json, detected_package_path).map do |item| + Dependency.from(item, detected_package_path) end end @@ -29,16 +31,16 @@ module LicenseFinder end def npm_json - stdout, _stderr, status = Dir.chdir(project_path) do + stdout, _stderr, status = within_project_path do shell.execute("npm list --json --long --production") end status.success? ? JSON.parse(stdout) : {} end def default_env - return {} unless shell.custom_certificate_installed? - - { 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.custom_certificate_path.to_s) } + { + 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.default_certificate_path).to_s + } end end end diff --git a/lib/license/finder/ext/nuget.rb b/lib/license/finder/ext/nuget.rb index 5e689d4..f292392 100644 --- a/lib/license/finder/ext/nuget.rb +++ b/lib/license/finder/ext/nuget.rb @@ -3,16 +3,14 @@ module LicenseFinder class Nuget def prepare - shell.execute(['apt-get', :install, '-y', '/opt/toolcache/mono*.deb']) create_vendor_path - Dir.chdir(project_path) do - shell.execute([ - '/opt/asdf/installs/mono/6.8.0.123/bin/cert-sync', - shell.default_certificate_path - ]) + + within_project_path do + tool_box.install(tool: :mono) + shell.execute([:cert_sync, shell.default_certificate_path]) shell.execute([ - '/opt/asdf/installs/mono/6.8.0.123/bin/mono', - '/opt/asdf/installs/mono/6.8.0.123/bin/nuget.exe', + :mono, + :nuget, :restore, detected_package_path, '-LockedMode', '-NoCache', @@ -48,7 +46,7 @@ module LicenseFinder ::License::Management::Nuspec.new(content).licenses end rescue StandardError => e - ::License::Management.logger.error(e) + log.error(e) [] end end diff --git a/lib/license/finder/ext/package_manager.rb b/lib/license/finder/ext/package_manager.rb index e343745..f90a8eb 100644 --- a/lib/license/finder/ext/package_manager.rb +++ b/lib/license/finder/ext/package_manager.rb @@ -6,10 +6,27 @@ module LicenseFinder ::License::Management.shell end + def log + ::License::Management.logger + end + + def tool_box + @tool_box ||= ::License::Management::ToolBox.new(shell, project_path) + end + + def prepare + create_vendor_path + return unless prepare_command + + within_project_path do + shell.execute(prepare_command) + end + end + def current_packages_with_relations current_packages rescue StandardError => e - ::License::Management.logger.error(e) + log.error(e) raise e unless @prepare_no_fail [] diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb index b329681..7ab8d4d 100644 --- a/lib/license/finder/ext/pip.rb +++ b/lib/license/finder/ext/pip.rb @@ -2,13 +2,6 @@ module LicenseFinder class Pip - def current_packages - return legacy_results unless virtual_env? - - dependencies = python.pip_licenses(detection_path: detected_package_path) - dependencies.any? ? dependencies : legacy_results - end - def possible_package_paths path = project_path || Pathname.pwd @@ -18,37 +11,52 @@ module LicenseFinder ] end - def prepare - return install_packages if detected_package_path == @requirements_path - - requirements_path = detected_package_path.dirname.join('requirements.txt') - requirements_path.write('.') unless requirements_path.exist? - install_packages + def installed?(*) + true end - private + def prepare + within_project_path do + return install_packages if detected_package_path == @requirements_path - def python - @python ||= ::License::Management::Python.new + requirements_path = detected_package_path.dirname.join('requirements.txt') + requirements_path.write('.') unless requirements_path.exist? + install_packages + end end - def install_packages + def current_packages within_project_path do - shell.execute(['virtualenv -p', python_executable, '--activators=bash --seeder=app-data .venv']) - shell.sh([". .venv/bin/activate", "&&", pip_install_command], env: python.default_env) + return legacy_results unless File.exist?('.venv/bin/activate') + + dependencies = pip_licenses.map do |dependency| + ::LicenseFinder::Dependency.new( + 'Pip', + dependency['Name'], + dependency['Version'], + description: dependency['Description'], + detection_path: detected_package_path, + homepage: dependency['URL'], + spec_licenses: [dependency['License']] + ) + end + dependencies.any? ? dependencies : legacy_results end end - def pip_install_command - [:pip, :install, '-v', '-i', python.pip_index_url, '-r', @requirements_path] - end + private - def python_executable - '"$(asdf where python)/bin/python"' + def python_version(env: ENV) + tool_box.version_of(:python, env: env) end - def virtual_env? - within_project_path { File.exist?('.venv/bin/activate') } + def install_packages + within_project_path do + tool_box.install(tool: :python, version: python_version, env: default_env) + + shell.execute(["/opt/asdf/installs/python/#{python_version}/bin/virtualenv", '-p', 'python', '--activators=bash --seeder=app-data .venv']) + shell.sh([". .venv/bin/activate", "&&", 'pip', 'install', '-v', '-r', @requirements_path], env: default_env) + end end def legacy_results @@ -58,12 +66,40 @@ module LicenseFinder 'Pip', name, version, - description: spec['description'], + description: spec['summary'] || spec['description'], detection_path: detected_package_path, homepage: spec['home_page'], spec_licenses: PipPackage.license_names_from_spec(spec) ) end end + + def pip_output + stdout, _, status = shell.execute([ + :asdf, :exec, :python, + LicenseFinder::BIN_PATH.join('license_finder_pip.py'), + detected_package_path + ], env: default_env) + return [] unless status.success? + + JSON.parse(stdout).map do |package| + package.values_at('name', 'version', 'dependencies', 'location') + end + end + + def pip_licenses + shell.sh([". .venv/bin/activate", '&&', 'pip', 'install', '--no-index', "--find-links /opt/gitlab/.config/virtualenv/app-data", 'pip-licenses'], env: default_env) + stdout, _, status = shell.sh([". .venv/bin/activate", '&&', 'pip-licenses', '--with-description', '--with-urls', '--format=json'], env: default_env) + status.success? ? JSON.parse(stdout[stdout.index('[')..-1]) : [] + end + + def default_env + @default_env ||= { + 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s, + 'PIP_DISABLE_PIP_VERSION_CHECK' => '1', + 'PIP_INDEX_URL' => ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/'), + 'PIP_NO_PYTHON_VERSION_WARNING' => '1' + } + end end end diff --git a/lib/license/finder/ext/pipenv.rb b/lib/license/finder/ext/pipenv.rb index 687c6fc..2dfad64 100644 --- a/lib/license/finder/ext/pipenv.rb +++ b/lib/license/finder/ext/pipenv.rb @@ -3,22 +3,43 @@ module LicenseFinder class Pipenv def prepare - return unless pipfile? + create_vendor_path + within_project_path do + return unless pipfile? - shell.execute([:pipenv, '--python', python.version], env: python.default_env) - shell.execute([:pipenv, :run, :pipenv, :sync, '--pypi-mirror', python.pip_index_url], env: python.default_env) + tool_box.install(tool: :python, version: python_version, env: default_env) + shell.execute([:asdf, :exec, :pipenv, '--python', python_version], env: default_env) + shell.execute([:asdf, :exec, :pipenv, :sync], env: default_env) + end end def current_packages - return legacy_results unless pipfile? + within_project_path do + return legacy_results unless pipfile? - python.pip_licenses(detection_path: detected_package_path) + dependencies = pip_licenses.map do |dependency| + ::LicenseFinder::Dependency.new( + 'Pip', + dependency['Name'], + dependency['Version'], + description: dependency['Description'], + detection_path: detected_package_path, + homepage: dependency['URL'], + spec_licenses: [dependency['License']] + ) + end + dependencies.any? ? dependencies : legacy_results + end end private - def python - @python ||= ::License::Management::Python.new + def python_version(env: ENV) + @python_version ||= + begin + version = lockfile_hash.dig('_meta', 'requires', 'python_version') + version ? tool_box.closest_match_to(tool: :python, version: version) || version : tool_box.version_of(:python, env: env) + end end def pipfile? @@ -38,7 +59,23 @@ module LicenseFinder end def lockfile_hash - @lockfile_hash ||= JSON.parse(IO.read(detected_package_path)) + @lockfile_hash ||= JSON.parse(detected_package_path.read) + end + + def pip_licenses + _, _, status = shell.sh([". .venv/bin/activate", '&&', 'pip', 'install', '--no-index', "--find-links /opt/gitlab/.config/virtualenv/app-data", 'pip-licenses'], env: default_env) + shell.sh([". .venv/bin/activate", '&&', 'pip', 'install', 'pip-licenses'], env: default_env) unless status.success? + stdout, _stderr, status = shell.sh([". .venv/bin/activate", '&&', 'pip-licenses', '--with-description', '--with-urls', '--format=json'], env: default_env) + status.success? ? JSON.parse(stdout[stdout.index('[')..-1]) : [] + end + + def default_env + @default_env ||= { + 'PIPENV_PYPI_MIRROR' => ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/'), + 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s, + 'PIP_DISABLE_PIP_VERSION_CHECK' => '1', + 'PIP_NO_PYTHON_VERSION_WARNING' => '1' + } end end end diff --git a/lib/license/finder/ext/yarn.rb b/lib/license/finder/ext/yarn.rb index 4ad97d4..e56f7d1 100644 --- a/lib/license/finder/ext/yarn.rb +++ b/lib/license/finder/ext/yarn.rb @@ -9,8 +9,19 @@ module LicenseFinder [project_path.join('yarn.lock')] end + def prepare + within_project_path do + tool_box.install(tool: :nodejs, version: nodejs_version, env: default_env) + shell.execute([ + :yarn, :install, + '--ignore-engines', '--ignore-scripts', + '--production' + ], env: default_env) + end + end + def current_packages - stdout, _stderr, status = Dir.chdir(project_path) do + stdout, _stderr, status = within_project_path do shell.execute(list_licenses_command) end return [] unless status.success? @@ -20,16 +31,6 @@ module LicenseFinder end end - def prepare - Dir.chdir(project_path) do - shell.execute([ - :yarn, :install, - '--ignore-engines', '--ignore-scripts', - '--production' - ], env: default_env) - end - end - private def list_licenses_command @@ -94,9 +95,13 @@ module LicenseFinder end def default_env - return {} unless shell.custom_certificate_installed? + { + 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.default_certificate_path).to_s + } + end - { 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.custom_certificate_path.to_s) } + def nodejs_version + @nodejs_version ||= tool_box.version_of(:nodejs) end end end diff --git a/lib/license/management.rb b/lib/license/management.rb index 41885d5..0b418e7 100644 --- a/lib/license/management.rb +++ b/lib/license/management.rb @@ -9,10 +9,10 @@ require 'license_finder' require 'license/management/loggable' require 'license/management/verifiable' require 'license/management/nuspec' -require 'license/management/python' require 'license/management/repository' require 'license/management/report' require 'license/management/shell' +require 'license/management/tool_box' require 'license/management/version' require 'license/finder/ext' diff --git a/lib/license/management/python.rb b/lib/license/management/python.rb deleted file mode 100644 index ede792e..0000000 --- a/lib/license/management/python.rb +++ /dev/null @@ -1,61 +0,0 @@ -# frozen_string_literal: true - -module License - module Management - class Python - attr_reader :shell - - def initialize(shell: ::License::Management.shell) - @shell = shell - end - - def major_version - version.split('.')[0] - end - - def version - ENV.fetch('ASDF_PYTHON_VERSION') do - _stdout, stderr, status = shell.execute([:python, '--version']) - status.success? ? stderr.split(' ')[-1] : 3 - end - end - - def pip_index_url - ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/') - end - - def pip_licenses(venv: '.venv', detection_path:) - _stdout, _stderr, status = shell.sh([ - ". #{venv}/bin/activate &&", - :pip, :install, - '--no-index', - '--find-links /opt/gitlab/.config/virtualenv/app-data', 'pip-licenses', '&&', - 'pip-licenses', - '--ignore-packages prettytable', - '--with-description', - '--with-urls', - '--from=meta', - '--format=json', - '--output-file pip-licenses.json' - ], env: { 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' }) - return [] unless status.success? - - JSON.parse(IO.read('pip-licenses.json')).map do |dependency| - ::LicenseFinder::Dependency.new( - 'Pip', - dependency['Name'], - dependency['Version'], - description: dependency['Description'], - detection_path: detection_path, - homepage: dependency['URL'], - spec_licenses: [dependency['License']] - ) - end - end - - def default_env - { 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s } - end - end - end -end diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb index 1a99895..b066bc1 100644 --- a/lib/license/management/shell.rb +++ b/lib/license/management/shell.rb @@ -4,12 +4,29 @@ module License module Management class Shell SPLIT_SCRIPT = "'BEGIN {x=0;} /BEGIN CERT/{x++} { print > \"custom.\" x \".crt\" }'" - attr_reader :default_certificate_path, :custom_certificate_path, :logger + COMMAND_MAP = { + asdf: '/opt/asdf/bin/asdf', + bundle: '/opt/asdf/bin/asdf exec bundle', + cargo: '/opt/asdf/bin/asdf exec cargo', + cert_sync: '/opt/asdf/installs/mono/6.8.0.123/bin/cert-sync', + embedded_bundle: '/opt/gitlab/embedded/bin/bundle', + gem: '/opt/asdf/bin/asdf exec gem', + go: '/opt/asdf/bin/asdf exec go', + keytool: '/opt/asdf/bin/asdf exec keytool', + mono: '/opt/asdf/installs/mono/6.8.0.123/bin/mono', + mvn: '/opt/asdf/bin/asdf exec mvn', + nuget: '/opt/asdf/installs/mono/6.8.0.123/bin/nuget.exe', + ruby: '/opt/asdf/bin/asdf exec ruby', + yarn: '/opt/asdf/bin/asdf exec yarn' + }.freeze + + attr_reader :default_env, :default_certificate_path, :custom_certificate_path, :logger def initialize(logger: License::Management.logger, certificate: ENV['ADDITIONAL_CA_CERT_BUNDLE']) @logger = logger @custom_certificate_path = Pathname.new('/usr/local/share/ca-certificates/custom.crt') @default_certificate_path = Pathname.new('/etc/ssl/certs/ca-certificates.crt') + @default_env = { 'SSL_CERT_FILE' => @default_certificate_path.to_s } trust!(certificate) if present?(certificate) end @@ -17,7 +34,7 @@ module License expanded_command = expand(command) collapsible_section(expanded_command) do logger.debug(expanded_command) - stdout, stderr, status = Open3.capture3(env, expanded_command) + stdout, stderr, status = Open3.capture3(default_env.merge(env), expanded_command) record(stdout, stderr, status) [stdout, stderr, status] end @@ -34,7 +51,10 @@ module License private def expand(command) - Array(command).flatten.map(&:to_s).join(' ') + Array(command) + .flatten + .map { |x| COMMAND_MAP.fetch(x, x).to_s } + .join(' ') end def trust!(certificate) @@ -44,37 +64,13 @@ module License execute('update-ca-certificates -v') Dir.glob('custom.*.crt').each do |path| - full_path = File.expand_path(path) - execute([:openssl, :x509, '-in', full_path, '-text', '-noout']) - execute(keytool_import_command(full_path)) - execute(keytool_list_command) + execute([:openssl, :x509, '-in', File.expand_path(path), '-text', '-noout']) end end execute([:cp, custom_certificate_path.to_s, "/usr/lib/ssl/certs/"]) execute([:c_rehash, '-v']) end - def keytool_import_command(file_path) - [ - :keytool, - '-importcert', - '-alias', Time.now.to_i, - '-file', file_path, - '-trustcacerts', - '-noprompt', - '-storepass', 'changeit', - '-keystore', keystore_path - ] - end - - def keytool_list_command - [:keytool, '-list', '-v', '-storepass changeit', '-keystore', keystore_path] - end - - def keystore_path - "#{ENV['JAVA_HOME']}/jre/lib/security/cacerts" - end - def present?(item) !item.nil? && !item.empty? end diff --git a/lib/license/management/tool_box.rb b/lib/license/management/tool_box.rb new file mode 100644 index 0000000..d2fe703 --- /dev/null +++ b/lib/license/management/tool_box.rb @@ -0,0 +1,123 @@ +# frozen_string_literal: true + +module License + module Management + class ToolBox + attr_reader :project_path, :shell + + def initialize(shell, project_path) + @shell = shell + @project_path = project_path + end + + def install(tool:, version: version_of(tool), env: {}) + Dir.chdir(project_path) do + deb = deb_for(tool, version) + if deb&.exist? + shell.execute([:dpkg, '-i', deb]) + elsif tool == :nodejs + shell.execute(['/opt/asdf/plugins/nodejs/bin/import-release-team-keyring']) + end + shell.execute([:asdf, :install, tool.to_s, version], env: env) + shell.execute([:asdf, :local, tool.to_s, version], env: env) + shell.execute([:asdf, :reshim], env: env) + end + + install_certificates_into_java_keystore(env) if tool == :java + end + + def version_of(tool, env: ENV) + Dir.chdir(project_path) do + case tool + when :java + java_version(env: env) + when :python + python_version(env: env) + else + asdf_detected_version_of(tool) + end + end + end + + def default_version_for(tool) + default_versions[tool.to_sym] + end + + def uninstall(tool:, version:) + Dir.chdir(project_path) do + return unless deb_for(tool, version)&.exist? + + shell.execute([:dpkg, '-r', "#{tool}-#{version}"]) + shell.execute([:asdf, :reshim]) + end + end + + def closest_match_to(tool:, version:) + deb = deb_for(tool, version) + deb.basename.to_s.gsub("#{tool}-", "").split('_', 2)[0] if deb&.exist? + end + + private + + def deb_for(tool, version) + Pathname.glob("/opt/toolcache/#{tool}-#{version}*.deb")[0] + end + + def version_error_for(tool) + /\Aversion\s(?<version>.+)\sis not installed for #{tool}\z/ + end + + def default_versions + @default_versions ||= + IO.read('/opt/gitlab/.tool-versions').each_line.each_with_object({}) do |line, memo| + name, version = line.chomp.split(' ', 2) + memo[name.to_sym] = version + end + end + + def asdf_detected_version_of(tool) + stdout, stderr, status = shell.execute([:asdf, :current, tool.to_s]) + + if status.success? + version, _ = stdout.split(' ', 2) + version + else + match = stderr.chomp.match(version_error_for(tool)) + match ? match[:version] : default_version_for(tool) + end + end + + def python_version(env:) + case env['LM_PYTHON_VERSION'] + when '3' + default_version_for(:python) + when '2' + closest_match_to(tool: :python, version: "2.") + else + major, minor, _rest = (env['ASDF_PYTHON_VERSION'] || asdf_detected_version_of(:python)).split('.') + closest_match_to(tool: :python, version: "#{major}.#{minor}") + end + end + + def java_version(env:) + lm_version = env['LM_JAVA_VERSION'] + return lm_version if lm_version && %w[8 11].include?(lm_version) + + asdf_version = env['ASDF_JAVA_VERSION'] + return asdf_version.gsub('adopt-openjdk', 'adoptopenjdk') if asdf_version + + asdf_detected_version_of(:java) + end + + def install_certificates_into_java_keystore(env) + Dir.chdir shell.custom_certificate_path.dirname do + Dir.glob('custom.*.crt').each do |path| + keystore_path = "#{env['JAVA_HOME']}/jre/lib/security/cacerts" + shell.execute([:keytool, '-importcert', '-alias', Time.now.to_i, '-file', File.expand_path(path), '-trustcacerts', '-noprompt', '-storepass', 'changeit', '-keystore', keystore_path], env: env) + shell.execute([:keytool, '-list', '-v', '-storepass changeit', '-keystore', keystore_path], env: env) + end + end + end + end + end +end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index fb91980..a9eefb3 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.23.0' + VERSION = '3.25.0' end end |
