Identify and resolve bottleneck with csv parsing

3 files changed, 24 insertions, 9 deletions

diff --git a/spec/integration/core/cache_spec.rb b/spec/integration/core/cache_spec.rb
index 06cdfa3..3f947b2 100644
--- a/spec/integration/core/cache_spec.rb
+++ b/spec/integration/core/cache_spec.rb
@@ -132,20 +132,20 @@ RSpec.describe Spandx::Core::Cache do
   end
 
   describe '#each' do
-    subject { described_class.new('rubygems', root: root_dir) }
+    context 'when a single item is present in the cache' do
+      subject { described_class.new('rubygems', root: root_dir) }
 
-    let(:root_dir) { Dir.mktmpdir }
+      let(:root_dir) { Dir.mktmpdir }
 
-    after do
-      FileUtils.remove_entry(root_dir)
-    end
-
-    context 'when a single item is present in the cache' do
       before do
         subject.insert('spandx', '0.0.0', ['MIT'])
       end
 
-      it 'yields each item in the index' do
+      after do
+        FileUtils.remove_entry(root_dir)
+      end
+
+      it 'yields each item in the cache' do
         collect = []
 
         subject.each do |item|
@@ -155,5 +155,19 @@ RSpec.describe Spandx::Core::Cache do
         expect(collect).to match_array([['spandx', '0.0.0', 'MIT']])
       end
     end
+
+    context 'when multiple items are in multiple datafiles' do
+      subject { described_class.new('rubygems', root: root_dir) }
+
+      let(:root_dir) { "#{Spandx.git[:rubygems].root}/.index" }
+
+      it 'yields each item in the cache' do
+        expect(subject.count).to be > 800_000
+      end
+
+      it 'yields each item quickly' do
+        expect { subject.take(100_000).count }.to perform_under(0.1).sample(10)
+      end
+    end
   end
 end
diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb
index 82f8c4c..63af672 100644
--- a/spec/spec_helper.rb
+++ b/spec/spec_helper.rb
@@ -3,6 +3,7 @@
 require 'bundler/setup'
 require 'spandx'
 
+require 'benchmark/ips'
 require 'parslet/convenience'
 require 'parslet/rig/rspec'
 require 'rspec-benchmark'
diff --git a/spec/unit/ruby/parsers/gemfile_lock_spec.rb b/spec/unit/ruby/parsers/gemfile_lock_spec.rb
index caa8341..cf75463 100644
--- a/spec/unit/ruby/parsers/gemfile_lock_spec.rb
+++ b/spec/unit/ruby/parsers/gemfile_lock_spec.rb
@@ -24,7 +24,7 @@ RSpec.describe Spandx::Ruby::Parsers::GemfileLock do
 
       specify { expect(spandx.name).to eql('spandx') }
       specify { expect(spandx.version).to eql(Spandx::VERSION) }
-      specify { expect(spandx.meta[:dependencies].map(&:name)).to match_array(%w[addressable bundler net-hippie nokogiri parslet thor zeitwerk]) }
+      specify { expect(spandx.meta[:dependencies].map(&:name)).to match_array(%w[addressable bundler fastest-csv net-hippie nokogiri parslet thor zeitwerk]) }
       specify { expect(spandx.meta[:platform]).to eql('ruby') }
       specify { expect(spandx.meta[:source]).to be_a_kind_of(Bundler::Source) }
     end