Merge branch '199059-setup-py' into 'master'v3.3.0

Use virtualenv and pip-licenses to scan python projects

See merge request gitlab-org/security-products/license-management!128

9 files changed, 99 insertions, 37 deletions

diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb
index 54b7d40..e83f64c 100644
--- a/lib/license/finder/ext/pip.rb
+++ b/lib/license/finder/ext/pip.rb
@@ -3,8 +3,19 @@
 module LicenseFinder
   class Pip
     def current_packages
-      detected_dependencies.map do |name, version|
-        PipPackage.new(name, version, pypi.definition_for(name, version))
+      return legacy_results unless virtual_env?
+
+      _stdout, _stderr, status = pip_licenses
+      return legacy_results unless status.success?
+
+      JSON.parse(IO.read('pip-licenses.json')).map do |dependency|
+        Package.new(
+          dependency['Name'],
+          dependency['Version'],
+          description: dependency['Description'],
+          homepage: dependency['URL'],
+          spec_licenses: [dependency['License']]
+        )
       end
     end
 
@@ -27,35 +38,49 @@ module LicenseFinder
 
     private
 
-    def detected_dependencies
-      stdout, _stderr, status = execute([
-        python_executable,
-        LicenseFinder::BIN_PATH.join('license_finder_pip.py'),
-        detected_package_path
-      ])
-      return [] unless status.success?
-
-      JSON.parse(stdout).map { |package| package.values_at('name', 'version') }
-    end
-
     def install_packages
-      execute([prepare_command, "-i", pip_index_url, "-r", @requirements_path])
+      within_project_dir do
+        shell.execute(['virtualenv -p', python_executable, '--activators=bash --seeder=app-data venv'])
+        shell.sh([". venv/bin/activate", "&&", :pip, :install, '-i', pip_index_url, '-r', @requirements_path])
+      end
     end
 
-    def execute(command)
-      Dir.chdir(project_path) do
-        ::LicenseFinder::SharedHelpers::Cmd.run(Array(command).join(' '))
-      end
+    def pip_licenses
+      shell.sh([
+        ". venv/bin/activate &&",
+        :pip, :install,
+        '--no-index',
+        '--find-links $HOME/.config/virtualenv/app-data', 'pip-licenses', '&&',
+        'pip-licenses',
+        '--ignore-packages prettytable',
+        '--with-description',
+        '--with-urls',
+        '--from=meta',
+        '--format=json',
+        '--output-file pip-licenses.json'
+      ], env: { 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' })
     end
 
     def python_executable
-      "python#{@python_version == '2' ? '' : '3'}"
+      '"$(asdf where python)/bin/python"'
     end
 
     def pip_index_url
       ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/')
     end
 
+    def virtual_env?
+      within_project_dir { File.exist?('venv/bin/activate') }
+    end
+
+    def within_project_dir
+      Dir.chdir(project_path) { yield }
+    end
+
+    def shell
+      @shell ||= ::License::Management::Shell.new
+    end
+
     def pypi
       @pypi ||= Spandx::Python::PyPI.new(sources: [
         Spandx::Python::Source.new({
@@ -65,5 +90,18 @@ module LicenseFinder
         })
       ])
     end
+
+    def legacy_results
+      pip_output.map do |name, version, children, location|
+        spec = pypi.definition_for(name, version)
+        Package.new(
+          name,
+          version,
+          description: spec['description'],
+          homepage: spec['home_page'],
+          spec_licenses: PipPackage.license_names_from_spec(spec)
+        )
+      end
+    end
   end
 end
diff --git a/lib/license/finder/ext/shared_helpers.rb b/lib/license/finder/ext/shared_helpers.rb
index b6b6fcd..cee79ab 100644
--- a/lib/license/finder/ext/shared_helpers.rb
+++ b/lib/license/finder/ext/shared_helpers.rb
@@ -4,11 +4,8 @@ module LicenseFinder
   module SharedHelpers
     class Cmd
       def self.run(command)
-        ::License::Management.logger.debug(command)
-        stdout, stderr, status = Open3.capture3(command)
-        ::License::Management.logger.debug(stdout) unless stdout.nil? || stdout.empty?
-        ::License::Management.logger.error(stderr) unless stderr.nil? || stderr.empty?
-        [stdout, stderr, status]
+        @shell ||= ::License::Management::Shell.new
+        @shell.execute(command)
       end
     end
   end
diff --git a/lib/license/management.rb b/lib/license/management.rb
index 16a9d62..e7a5b23 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -11,6 +11,7 @@ require 'license/management/loggable'
 require 'license/management/verifiable'
 require 'license/management/repository'
 require 'license/management/report'
+require 'license/management/shell'
 require 'license/management/version'
 
 require 'license/finder/ext'
diff --git a/lib/license/management/loggable.rb b/lib/license/management/loggable.rb
index 0122018..37bcf37 100644
--- a/lib/license/management/loggable.rb
+++ b/lib/license/management/loggable.rb
@@ -6,14 +6,6 @@ module License
       def logger
         License::Management.logger
       end
-
-      def log_info(message)
-        logger.info(message)
-      end
-
-      def log_error(message)
-        logger.error(message)
-      end
     end
   end
 end
diff --git a/lib/license/management/report/v1.rb b/lib/license/management/report/v1.rb
index 49423c6..27495b5 100644
--- a/lib/license/management/report/v1.rb
+++ b/lib/license/management/report/v1.rb
@@ -31,7 +31,7 @@ module License
           license = { name: join_license_names(dependency.licenses) }
 
           urls = dependency.licenses.map(&:url).reject { |x| blank?(x) }.uniq.sort
-          log_info("multiple urls detected: #{urls.inspect}") if urls.size > 1
+          logger.info("multiple urls detected: #{urls.inspect}") if urls.size > 1
           url = urls[0] || license_data(dependency.licenses.first)['url']
 
           license[:url] = url if present?(url)
diff --git a/lib/license/management/report/v2.rb b/lib/license/management/report/v2.rb
index 6ab6b99..f8c96da 100644
--- a/lib/license/management/report/v2.rb
+++ b/lib/license/management/report/v2.rb
@@ -31,7 +31,7 @@ module License
 
         def map_from(dependency)
           licenses = dependency.licenses.map { |license| data_for(license)['id'] }.sort
-          log_info [dependency.name, dependency.version, licenses].inspect
+          logger.info [dependency.name, dependency.version, licenses].inspect
 
           {
             name: dependency.name,
diff --git a/lib/license/management/repository.rb b/lib/license/management/repository.rb
index b13cec8..fdd4eae 100644
--- a/lib/license/management/repository.rb
+++ b/lib/license/management/repository.rb
@@ -60,7 +60,7 @@ module License
       end
 
       def generate_item_for(license)
-        log_info("Detected unknown license `#{license.short_name}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.")
+        logger.info("Detected unknown license `#{license.short_name}`. Contribute to https://gitlab.com/gitlab-org/security-products/license-management#contributing.")
         name = take_first_line_from(license.name)
         {
           'id' => name.downcase,
@@ -88,7 +88,7 @@ module License
 
         uri.path.split('/')[-1]
       rescue StandardError => e
-        log_info(e)
+        logger.error(e)
         nil
       end
     end
diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb
new file mode 100644
index 0000000..903d0b6
--- /dev/null
+++ b/lib/license/management/shell.rb
@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module License
+  module Management
+    class Shell
+      attr_reader :logger
+
+      def initialize(logger: License::Management.logger)
+        @logger = logger
+      end
+
+      def execute(command, env: {})
+        expanded_command = expand(command)
+        logger.debug(expanded_command)
+
+        stdout, stderr, status = Open3.capture3(env, expanded_command)
+
+        logger.debug(stdout) unless stdout.nil? || stdout.empty?
+        logger.error(stderr) unless stderr.nil? || stderr.empty?
+        [stdout, stderr, status]
+      end
+
+      def sh(command, env: {})
+        execute("sh -c '#{expand(command)}'", env: env)
+      end
+
+      private
+
+      def expand(command)
+        Array(command).map(&:to_s).join(' ')
+      end
+    end
+  end
+end
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
index 946d5e9..22f92ca 100644
--- a/lib/license/management/version.rb
+++ b/lib/license/management/version.rb
@@ -2,6 +2,6 @@
 
 module License
   module Management
-    VERSION = '3.2.0'
+    VERSION = '3.3.0'
   end
 end