Merge pull request #29 from spandx/streaming

Stream results to stdout

22 files changed, 214 insertions, 152 deletions

diff --git a/lib/spandx.rb b/lib/spandx.rb
index b7a5bfa..78fbdcb 100644
--- a/lib/spandx.rb
+++ b/lib/spandx.rb
@@ -8,11 +8,11 @@ require 'json'
 require 'logger'
 require 'net/hippie'
 require 'nokogiri'
+require 'oj'
 require 'parslet'
 require 'pathname'
 require 'yaml'
 require 'zeitwerk'
-require 'terminal-table'
 require 'spandx/spandx'
 
 loader = Zeitwerk::Loader.for_gem
diff --git a/lib/spandx/cli.rb b/lib/spandx/cli.rb
index 817ff89..e5a8d76 100644
--- a/lib/spandx/cli.rb
+++ b/lib/spandx/cli.rb
@@ -1,8 +1,8 @@
 # frozen_string_literal: true
 
-require 'nanospinner'
 require 'thor'
-require 'tty-screen'
+require 'tty-spinner'
+require 'terminal-table'
 
 module Spandx
   module Cli
diff --git a/lib/spandx/cli/commands/scan.rb b/lib/spandx/cli/commands/scan.rb
index 9e874f3..22420fc 100644
--- a/lib/spandx/cli/commands/scan.rb
+++ b/lib/spandx/cli/commands/scan.rb
@@ -4,51 +4,49 @@ module Spandx
   module Cli
     module Commands
       class Scan
-        attr_reader :scan_path, :spinner
+        include Spandx::Core
+        attr_reader :scan_path
 
         def initialize(scan_path, options)
           @scan_path = ::Pathname.new(scan_path)
           @options = options
-          @spinner = options[:show_progress] ? ::Spandx::Core::Spinner.new : ::Spandx::Core::Spinner::NULL
           require(options[:require]) if options[:require]
         end
 
         def execute(output: $stdout)
-          report = ::Spandx::Core::Report.new
-          each_file do |file|
-            spinner.spin(file)
-            each_dependency_from(file) do |dependency|
-              spinner.spin(file)
-              report.add(dependency)
+          with_printer(output) do |printer|
+            each_dependency do |dependency|
+              printer.print_line(Plugin.enhance(dependency), output)
             end
           end
-          spinner.stop
-          output.puts(format(report.to(@options[:format])))
         end
 
         private
 
         def each_file
-          Spandx::Core::PathTraversal
+          PathTraversal
             .new(scan_path, recursive: @options[:recursive])
             .each { |file| yield file }
         end
 
-        def each_dependency_from(file)
-          ::Spandx::Core::Parser
-            .parse(file)
-            .map { |x| enhance(x) }
-            .each { |dependency| yield dependency }
+        def each_dependency
+          each_file do |file|
+            Parser.parse(file).each do |dependency|
+              yield dependency
+            end
+          end
         end
 
         def format(output)
           Array(output).map(&:to_s)
         end
 
-        def enhance(dependency)
-          ::Spandx::Core::Plugin
-            .all
-            .inject(dependency) { |memo, plugin| plugin.enhance(memo) }
+        def with_printer(output)
+          printer = ::Spandx::Cli::Printer.for(@options[:format])
+          printer.print_header(output)
+          yield printer
+        ensure
+          printer.print_footer(output)
         end
       end
     end
diff --git a/lib/spandx/cli/main.rb b/lib/spandx/cli/main.rb
index f22dd34..bb27f83 100644
--- a/lib/spandx/cli/main.rb
+++ b/lib/spandx/cli/main.rb
@@ -8,14 +8,14 @@ module Spandx
       method_option :recursive, aliases: '-R', type: :boolean, desc: 'Perform recursive scan', default: false
       method_option :airgap, aliases: '-a', type: :boolean, desc: 'Disable network connections', default: false
       method_option :logfile, aliases: '-l', type: :string, desc: 'Path to a logfile', default: '/dev/null'
-      method_option :format, aliases: '-f', type: :string, desc: 'Format of report. (table, csv, json, hash)', default: 'table'
+      method_option :format, aliases: '-f', type: :string, desc: 'Format of report. (table, csv, json)', default: 'table'
       method_option :pull, aliases: '-p', type: :boolean, desc: 'Pull the latest cache before the scan', default: false
       method_option :require, aliases: '-r', type: :string, desc: 'Causes spandx to load the library using require.', default: nil
-      method_option :show_progress, aliases: '-sp', type: :boolean, desc: 'Shows a progress bar', default: true
       def scan(lockfile = Pathname.pwd)
         if options[:help]
           invoke :help, ['scan']
         else
+          Oj.default_options = { mode: :strict }
           Spandx.airgap = options[:airgap]
           Spandx.logger = Logger.new(options[:logfile])
           pull if options[:pull]
diff --git a/lib/spandx/cli/printer.rb b/lib/spandx/cli/printer.rb
new file mode 100644
index 0000000..f21be93
--- /dev/null
+++ b/lib/spandx/cli/printer.rb
@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    class Printer
+      def match?(_format)
+        raise ::Spandx::Error, :match?
+      end
+
+      def print_header(io); end
+
+      def print_line(dependency, io)
+        io.puts(dependency.to_s)
+      end
+
+      def print_footer(io); end
+
+      class << self
+        include Core::Registerable
+
+        def for(format)
+          find { |x| x.match?(format) } || new
+        end
+      end
+    end
+  end
+end
diff --git a/lib/spandx/cli/printers/csv.rb b/lib/spandx/cli/printers/csv.rb
new file mode 100644
index 0000000..ab73960
--- /dev/null
+++ b/lib/spandx/cli/printers/csv.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Csv < Printer
+        def match?(format)
+          format.to_sym == :csv
+        end
+
+        def print_line(dependency, io)
+          io.puts(CSV.generate_line(dependency.to_a))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/spandx/cli/printers/json.rb b/lib/spandx/cli/printers/json.rb
new file mode 100644
index 0000000..b796d89
--- /dev/null
+++ b/lib/spandx/cli/printers/json.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Json < Printer
+        def match?(format)
+          format.to_sym == :json
+        end
+
+        def print_line(dependency, io)
+          io.puts(Oj.dump(dependency.to_h))
+        end
+      end
+    end
+  end
+end
diff --git a/lib/spandx/cli/printers/table.rb b/lib/spandx/cli/printers/table.rb
new file mode 100644
index 0000000..931b739
--- /dev/null
+++ b/lib/spandx/cli/printers/table.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Cli
+    module Printers
+      class Table < Printer
+        HEADINGS = ['Name', 'Version', 'Licenses', 'Location'].freeze
+
+        def initialize
+          @spinner = TTY::Spinner.new(output: $stderr)
+        end
+
+        def match?(format)
+          format.to_sym == :table
+        end
+
+        def print_header(_io)
+          @dependencies = SortedSet.new
+          @spinner.auto_spin
+        end
+
+        def print_line(dependency, _io)
+          @dependencies << dependency
+        end
+
+        def print_footer(io)
+          @spinner.stop
+          io.puts(to_table(@dependencies.map(&:to_a)))
+        end
+
+        private
+
+        def to_table(rows)
+          Terminal::Table.new(headings: HEADINGS) do |table|
+            rows.each { |row| table.add_row(row) }
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/spandx/core/git.rb b/lib/spandx/core/git.rb
index f98734d..25716ab 100644
--- a/lib/spandx/core/git.rb
+++ b/lib/spandx/core/git.rb
@@ -11,9 +11,8 @@ module Spandx
       end
 
       def read(path)
-        full_path = File.join(root, path)
-
-        IO.read(full_path) if File.exist?(full_path)
+        full_path = root.join(path)
+        full_path.read if full_path.exist?
       end
 
       def update!
@@ -25,15 +24,16 @@ module Spandx
       def path_for(url)
         uri = URI.parse(url)
         name = uri.path.gsub(/\.git$/, '')
-        File.expand_path(File.join(Dir.home, '.local', 'share', name))
+        Pathname(File.expand_path(File.join(Dir.home, '.local', 'share', name)))
       end
 
       def dotgit?
-        File.directory?(File.join(root, '.git'))
+        root.join('.git').directory?
       end
 
       def clone!
-        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root)
+        system('rm', '-rf', root.to_s) if root.exist?
+        system('git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', 'master', url, root.to_s)
       end
 
       def pull!
diff --git a/lib/spandx/core/plugin.rb b/lib/spandx/core/plugin.rb
index f4841cf..6564c60 100644
--- a/lib/spandx/core/plugin.rb
+++ b/lib/spandx/core/plugin.rb
@@ -9,6 +9,12 @@ module Spandx
 
       class << self
         include Registerable
+
+        def enhance(dependency)
+          Plugin.all.inject(dependency) do |memo, plugin|
+            plugin.enhance(memo)
+          end
+        end
       end
     end
   end
diff --git a/lib/spandx/core/report.rb b/lib/spandx/core/report.rb
deleted file mode 100644
index dc57f6e..0000000
--- a/lib/spandx/core/report.rb
+++ /dev/null
@@ -1,54 +0,0 @@
-# frozen_string_literal: true
-
-module Spandx
-  module Core
-    class Report
-      attr_reader :dependencies
-
-      FORMATS = {
-        csv: :to_csv,
-        hash: :to_h,
-        json: :to_json,
-        table: :to_table,
-      }.freeze
-
-      def initialize
-        @dependencies = SortedSet.new
-      end
-
-      def add(dependency)
-        @dependencies << dependency
-      end
-
-      def to(format, formats: FORMATS)
-        public_send(formats.fetch(format&.to_sym, :to_json))
-      end
-
-      def to_table
-        Terminal::Table.new(headings: ['Name', 'Version', 'Licenses', 'Location']) do |t|
-          dependencies.each do |d|
-            t.add_row d.to_a
-          end
-        end
-      end
-
-      def to_h
-        { version: '1.0', dependencies: [] }.tap do |report|
-          dependencies.each do |dependency|
-            report[:dependencies].push(dependency.to_h)
-          end
-        end
-      end
-
-      def to_json(*_args)
-        JSON.pretty_generate(to_h)
-      end
-
-      def to_csv
-        dependencies.map do |dependency|
-          CSV.generate_line(dependency.to_a)
-        end
-      end
-    end
-  end
-end
diff --git a/lib/spandx/core/spinner.rb b/lib/spandx/core/spinner.rb
deleted file mode 100644
index af3c7a7..0000000
--- a/lib/spandx/core/spinner.rb
+++ /dev/null
@@ -1,51 +0,0 @@
-# frozen_string_literal: true
-
-module Spandx
-  module Core
-    class Spinner
-      NULL = Class.new do
-        def self.spin(*args); end
-
-        def self.stop(*args); end
-      end
-
-      attr_reader :columns, :spinner
-
-      def initialize(columns: TTY::Screen.columns, output: $stderr)
-        @columns = columns
-        @spinner = Nanospinner.new(output)
-        @queue = Queue.new
-        @thread = Thread.new { work }
-      end
-
-      def spin(message)
-        @queue.enq(justify(message))
-        yield if block_given?
-      end
-
-      def stop
-        @queue.clear
-        @queue.enq(:stop)
-        @thread.join
-      end
-
-      private
-
-      def justify(message)
-        message.to_s.ljust(columns - 3)
-      end
-
-      def work
-        last_message = justify('')
-        loop do
-          message = @queue.empty? ? last_message : @queue.deq
-          break if message == :stop
-
-          spinner.spin(message)
-          last_message = message
-          sleep 0.1
-        end
-      end
-    end
-  end
-end
diff --git a/lib/spandx/core/thread_pool.rb b/lib/spandx/core/thread_pool.rb
new file mode 100644
index 0000000..fdbced5
--- /dev/null
+++ b/lib/spandx/core/thread_pool.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Spandx
+  module Core
+    class ThreadPool
+      def initialize(size: 1)
+        @size = size
+        @queue = Queue.new
+        @pool = size.times.map { start_worker_thread(@queue) }
+      end
+
+      def run(*args, &job)
+        @queue.enq([job, args])
+      end
+
+      def done?
+        @queue.empty?
+      end
+
+      def shutdown
+        @size.times do
+          run { throw :exit }
+        end
+
+        @pool.map(&:join)
+      end
+
+      def self.open(**args)
+        pool = new(**args)
+        yield pool
+      ensure
+        pool.shutdown
+      end
+
+      private
+
+      def start_worker_thread(queue)
+        Thread.new(queue) do |q|
+          catch(:exit) do
+            loop do
+              job, args = q.deq
+              job.call(args)
+            end
+          end
+        end
+      end
+    end
+  end
+end
diff --git a/lib/spandx/dotnet/nuget_gateway.rb b/lib/spandx/dotnet/nuget_gateway.rb
index 6776553..089d2bb 100644
--- a/lib/spandx/dotnet/nuget_gateway.rb
+++ b/lib/spandx/dotnet/nuget_gateway.rb
@@ -69,7 +69,7 @@ module Spandx
 
       def fetch_json(url)
         response = http.get(url)
-        http.ok?(response) ? JSON.parse(response.body) : {}
+        http.ok?(response) ? Oj.load(response.body) : {}
       end
 
       def fetch_xml(url)
diff --git a/lib/spandx/js/parsers/npm.rb b/lib/spandx/js/parsers/npm.rb
index b92e1a2..2402454 100644
--- a/lib/spandx/js/parsers/npm.rb
+++ b/lib/spandx/js/parsers/npm.rb
@@ -18,8 +18,8 @@ module Spandx
 
         private
 
-        def each_metadata(file_path)
-          package_lock = JSON.parse(IO.read(file_path))
+        def each_metadata(path)
+          package_lock = Oj.load(path.read)
           package_lock['dependencies'].each do |name, metadata|
             yield metadata.merge('name' => name)
           end
diff --git a/lib/spandx/js/yarn_pkg.rb b/lib/spandx/js/yarn_pkg.rb
index 19a8dd3..bb479b4 100644
--- a/lib/spandx/js/yarn_pkg.rb
+++ b/lib/spandx/js/yarn_pkg.rb
@@ -27,7 +27,7 @@ module Spandx
         response = http.get(uri, escape: false)
 
         if http.ok?(response)
-          json = JSON.parse(response.body)
+          json = Oj.load(response.body)
           json['versions'] ? json['versions'][dependency.version] : json
         else
           {}
diff --git a/lib/spandx/php/packagist_gateway.rb b/lib/spandx/php/packagist_gateway.rb
index 88ec6d3..8580072 100644
--- a/lib/spandx/php/packagist_gateway.rb
+++ b/lib/spandx/php/packagist_gateway.rb
@@ -17,7 +17,7 @@ module Spandx
         response = http.get("https://repo.packagist.org/p/#{dependency.name}.json")
         return [] unless http.ok?(response)
 
-        json = JSON.parse(response.body)
+        json = Oj.load(response.body)
         json['packages'][dependency.name][dependency.version]['license']
       end
     end
diff --git a/lib/spandx/php/parsers/composer.rb b/lib/spandx/php/parsers/composer.rb
index 61c29a8..e1bb240 100644
--- a/lib/spandx/php/parsers/composer.rb
+++ b/lib/spandx/php/parsers/composer.rb
@@ -10,7 +10,7 @@ module Spandx
 
         def parse(path)
           items = Set.new
-          composer_lock = JSON.parse(path.read)
+          composer_lock = Oj.load(path.read)
           composer_lock['packages'].concat(composer_lock['packages-dev']).each do |dependency|
             items.add(map_from(path, dependency))
           end
diff --git a/lib/spandx/python/parsers/pipfile_lock.rb b/lib/spandx/python/parsers/pipfile_lock.rb
index 08b8644..e3ea58d 100644
--- a/lib/spandx/python/parsers/pipfile_lock.rb
+++ b/lib/spandx/python/parsers/pipfile_lock.rb
@@ -19,7 +19,7 @@ module Spandx
         private
 
         def dependencies_from(lockfile)
-          json = JSON.parse(lockfile.read)
+          json = Oj.load(lockfile.read)
           each_dependency(json) do |name, version|
             yield ::Spandx::Core::Dependency.new(
               path: lockfile,
diff --git a/lib/spandx/python/pypi.rb b/lib/spandx/python/pypi.rb
index 7849f29..ef3080a 100644
--- a/lib/spandx/python/pypi.rb
+++ b/lib/spandx/python/pypi.rb
@@ -49,19 +49,26 @@ module Spandx
       end
 
       def version_from(url)
-        path = SUBSTITUTIONS.inject(URI.parse(url).path.split('/')[-1]) do |memo, item|
-          memo.gsub(item, '')
-        end
-
+        path = cleanup(url)
         return if path.rindex('-').nil?
 
-        path.scan(/-\d+\..*/)[-1][1..-1]
+        section = path.scan(/-\d+\..*/)
+        section = path.scan(/-\d+\.?.*/) if section.empty?
+        section[-1][1..-1]
+      rescue StandardError => error
+        warn([url, error].inspect)
       end
 
       private
 
       attr_reader :http
 
+      def cleanup(url)
+        SUBSTITUTIONS.inject(URI.parse(url).path.split('/')[-1]) do |memo, item|
+          memo.gsub(item, '')
+        end
+      end
+
       def sources_for(dependency)
         return default_sources if dependency.meta.empty?
 
@@ -76,7 +83,7 @@ module Spandx
         sources.each do |source|
           html_from(source, '/simple/').css('a[href*="/simple"]').each do |node|
             each_version(source, node[:href]) do |dependency|
-              definition = source.lookup(dependency[:name], dependency[:version])
+              definition = source.lookup(dependency[:name], dependency[:version], http: http)
               yield dependency.merge(license: definition['license'])
             end
           end
@@ -93,7 +100,12 @@ module Spandx
 
       def html_from(source, path)
         url = URI.join(source.uri.to_s, path).to_s
-        Nokogiri::HTML(http.get(url).body)
+        response = http.get(url)
+        if http.ok?(response)
+          Nokogiri::HTML(response.body)
+        else
+          Nokogiri::HTML('<html><head></head><body></body></html>')
+        end
       end
     end
   end
diff --git a/lib/spandx/python/source.rb b/lib/spandx/python/source.rb
index af0cbbb..2379c16 100644
--- a/lib/spandx/python/source.rb
+++ b/lib/spandx/python/source.rb
@@ -22,7 +22,7 @@ module Spandx
       def lookup(name, version, http: Spandx.http)
         response = http.get(uri_for(name, version))
         if http.ok?(response)
-          JSON.parse(response.body)
+          Oj.load(response.body)
         else
           {}
         end
diff --git a/lib/spandx/ruby/gateway.rb b/lib/spandx/ruby/gateway.rb
index 54e8107..208eb9e 100644
--- a/lib/spandx/ruby/gateway.rb
+++ b/lib/spandx/ruby/gateway.rb
@@ -27,7 +27,7 @@ module Spandx
       end
 
       def parse(json)
-        JSON.parse(json)
+        Oj.load(json)
       end
     end
   end