From 99c33d2b132979bb0af0faabfa3049928de8c431 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 09:54:11 -0600 Subject: Compress /opt/asdf using level 19 in zstandard --- config/install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/install.sh b/config/install.sh index 39a3197..6f684c6 100644 --- a/config/install.sh +++ b/config/install.sh @@ -160,7 +160,8 @@ rm -fr "$ASDF_DATA_DIR/docs" \ /var/log/* cd /opt -tar --use-compress-program zstd -cf /opt/asdf.tar.zst asdf +tar -cf /opt/asdf.tar asdf +zstd -19 -k asdf.tar -o asdf.tar.zst rm -fr /opt/asdf/ cd /usr/lib -- cgit v1.2.3 From b4b715f6b5cc45425b9b61a1b9f47f95a63a2795 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 10:33:44 -0600 Subject: Update v3.8.1 CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c3459b0..5f29c22 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ ## v3.8.1 - Exclude `devDependencies` from `yarn` scan report. (!147) +- Remove `spandx` dependency and bring back Ruby 2.4+ support. (!147) ## v3.8.0 -- cgit v1.2.3 From a2760dd972af7f225677b12fd92c31ad991dd9cd Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 10:40:38 -0600 Subject: Trigger downstream pipeline --- .gitlab/release.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitlab/release.yml b/.gitlab/release.yml index 44cc42e..3bb7d3c 100644 --- a/.gitlab/release.yml +++ b/.gitlab/release.yml @@ -9,6 +9,9 @@ major: extends: .release variables: IMAGE_TAG: $MAJOR + trigger: + project: gitlab-org/security-products/analyzers/license-finder + branch: master latest: extends: .release -- cgit v1.2.3 From 5e6cad59f389ea4c32eb760e7235e6f1ac6cdcc9 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 10:49:51 -0600 Subject: Add spec for bower project --- spec/fixtures/js/bower/bower.json | 12 ++++++++++++ spec/integration/js/bower_spec.rb | 13 +++++++++++++ 2 files changed, 25 insertions(+) create mode 100644 spec/fixtures/js/bower/bower.json create mode 100644 spec/integration/js/bower_spec.rb diff --git a/spec/fixtures/js/bower/bower.json b/spec/fixtures/js/bower/bower.json new file mode 100644 index 0000000..9a905fd --- /dev/null +++ b/spec/fixtures/js/bower/bower.json @@ -0,0 +1,12 @@ +{ + "name": "bower", + "license": "MIT", + "private": true, + "ignore": [ + "**/.*", + "node_modules", + "bower_components", + "test", + "tests" + ] +} diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb new file mode 100644 index 0000000..d2fb682 --- /dev/null +++ b/spec/integration/js/bower_spec.rb @@ -0,0 +1,13 @@ +require 'spec_helper' + +RSpec.describe "bower" do + context "when scanning a simple bower project" do + subject { runner.scan } + + before do + runner.add_file('bower.json', fixture_file_content('js/bower/bower.json')) + end + + specify { expect(subject).to match_schema(version: '2.0') } + end +end -- cgit v1.2.3 From ea84a8f5badf153744486c51a75b1f492763fdfd Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 11:06:04 -0600 Subject: Add bower dependencies and expected assertions --- spec/fixtures/js/bower/bower.json | 8 +++++++- spec/integration/js/bower_spec.rb | 5 +++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/spec/fixtures/js/bower/bower.json b/spec/fixtures/js/bower/bower.json index 9a905fd..4cfbefc 100644 --- a/spec/fixtures/js/bower/bower.json +++ b/spec/fixtures/js/bower/bower.json @@ -8,5 +8,11 @@ "bower_components", "test", "tests" - ] + ], + "dependencies": { + "jquery": "^3.5.1", + "masonry-layout": "desandro/masonry#^4.2.2", + "cli": "git://github.com/npm/cli.git#^6.14.5", + "stimulus.umd": "https://unpkg.com/stimulus/dist/stimulus.umd.js" + } } diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb index d2fb682..68ab12d 100644 --- a/spec/integration/js/bower_spec.rb +++ b/spec/integration/js/bower_spec.rb @@ -9,5 +9,10 @@ RSpec.describe "bower" do end specify { expect(subject).to match_schema(version: '2.0') } + specify { expect(subject.dependency_names).to match_array(['cli', 'jquery', 'masonry-layout', 'stimulus.umd']) } + specify { expect(subject.licenses_for('cli')).to match_array(['Artistic-2.0']) } + specify { expect(subject.licenses_for('jquery')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('masonry-layout')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('stimulus.umd')).to match_array(['MIT']) } end end -- cgit v1.2.3 From 0f1db0d60c61a6db1c5389fc0ddb2d984789b330 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 11:42:07 -0600 Subject: Add --allow-root option to install step --- lib/license/finder/ext.rb | 1 + lib/license/finder/ext/bower.rb | 55 +++++++++++++++++++++++++++++++++++++++ spec/integration/js/bower_spec.rb | 4 +-- 3 files changed, 58 insertions(+), 2 deletions(-) create mode 100644 lib/license/finder/ext/bower.rb diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb index 3d8a463..3c56c7a 100644 --- a/lib/license/finder/ext.rb +++ b/lib/license/finder/ext.rb @@ -1,5 +1,6 @@ # frozen_string_literal: true +require 'license/finder/ext/bower' require 'license/finder/ext/go_modules' require 'license/finder/ext/gradle' require 'license/finder/ext/license' diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb new file mode 100644 index 0000000..3c6ba6f --- /dev/null +++ b/lib/license/finder/ext/bower.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module LicenseFinder + class Bower < PackageManager + def prepare + shell.execute([:bower, :install, '--allow-root']) + end + + def current_packages + bower_output.map do |bower_module| + map_from(bower_module) + end + end + + def possible_package_paths + [project_path.join('bower.json')] + end + + private + + def bower_output + stdout, _stderr, status = Dir.chdir(project_path) do + shell.execute([:bower, :list, '--json', '-l', 'action', '--allow-root']) + end + return [] unless status.success? + + json = JSON.parse(stdout) + json.fetch('dependencies', {}).values + end + + def map_from(bower_module) + spec = bower_module.fetch('pkgMeta', {}) + + if spec.empty? + endpoint = bower_module.fetch('endpoint', {}) + Package.new( + endpoint['name'], + endpoint['target'], + install_path: bower_module['canonicalDir'], + ) + else + Package.new( + spec['name'], + spec['version'], + summary: spec['description'], + description: spec['readme'], + homepage: spec['homepage'], + spec_licenses: Package.license_names_from_standard_spec(spec), + install_path: bower_module['canonicalDir'], + ) + end + end + end +end + diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb index 68ab12d..b72f256 100644 --- a/spec/integration/js/bower_spec.rb +++ b/spec/integration/js/bower_spec.rb @@ -10,9 +10,9 @@ RSpec.describe "bower" do specify { expect(subject).to match_schema(version: '2.0') } specify { expect(subject.dependency_names).to match_array(['cli', 'jquery', 'masonry-layout', 'stimulus.umd']) } - specify { expect(subject.licenses_for('cli')).to match_array(['Artistic-2.0']) } + specify { expect(subject.licenses_for('cli')).to match_array(['Apache-2.0', 'BSD-3-Clause', 'ISC', 'MIT']) } specify { expect(subject.licenses_for('jquery')).to match_array(['MIT']) } specify { expect(subject.licenses_for('masonry-layout')).to match_array(['MIT']) } - specify { expect(subject.licenses_for('stimulus.umd')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('stimulus.umd')).to match_array(['unknown']) } end end -- cgit v1.2.3 From ee7b5af4bd860f373cb3744329152f9c837f3cd1 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 12:07:27 -0600 Subject: Collect dependencies of dependencies in bower project --- lib/license/finder/ext/bower.rb | 18 ++++++++++-------- spec/integration/js/bower_spec.rb | 7 ++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb index 3c6ba6f..53d5f5e 100644 --- a/lib/license/finder/ext/bower.rb +++ b/lib/license/finder/ext/bower.rb @@ -8,8 +8,8 @@ module LicenseFinder def current_packages bower_output.map do |bower_module| - map_from(bower_module) - end + map_all(bower_module) + end.flatten end def possible_package_paths @@ -28,28 +28,30 @@ module LicenseFinder json.fetch('dependencies', {}).values end - def map_from(bower_module) + def map_all(bower_module) + items = [] spec = bower_module.fetch('pkgMeta', {}) if spec.empty? endpoint = bower_module.fetch('endpoint', {}) - Package.new( + items << Package.new( endpoint['name'], endpoint['target'], - install_path: bower_module['canonicalDir'], + install_path: bower_module['canonicalDir'] ) else - Package.new( + items << Package.new( spec['name'], spec['version'], summary: spec['description'], description: spec['readme'], homepage: spec['homepage'], spec_licenses: Package.license_names_from_standard_spec(spec), - install_path: bower_module['canonicalDir'], + install_path: bower_module['canonicalDir'] ) end + + items + bower_module.fetch('dependencies', {}).values.map { |x| map_all(x) } end end end - diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb index b72f256..1f97645 100644 --- a/spec/integration/js/bower_spec.rb +++ b/spec/integration/js/bower_spec.rb @@ -9,10 +9,15 @@ RSpec.describe "bower" do end specify { expect(subject).to match_schema(version: '2.0') } - specify { expect(subject.dependency_names).to match_array(['cli', 'jquery', 'masonry-layout', 'stimulus.umd']) } + specify { expect(subject.dependency_names).to match_array(['cli', 'ev-emitter', 'fizzy-ui-utils', 'get-size', 'jquery', 'masonry-layout', 'matches-selector', 'outlayer', 'stimulus.umd']) } specify { expect(subject.licenses_for('cli')).to match_array(['Apache-2.0', 'BSD-3-Clause', 'ISC', 'MIT']) } + specify { expect(subject.licenses_for('ev-emitter')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('fizzy-ui-utils')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('get-size')).to match_array(['MIT']) } specify { expect(subject.licenses_for('jquery')).to match_array(['MIT']) } specify { expect(subject.licenses_for('masonry-layout')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('matches-selector')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('outlayer')).to match_array(['MIT']) } specify { expect(subject.licenses_for('stimulus.umd')).to match_array(['unknown']) } end end -- cgit v1.2.3 From 8805762269ed3d93197562eb4af8a56e9be00863 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 12:28:08 -0600 Subject: Flatten dependencies --- lib/license/finder/ext/bower.rb | 52 ++++++++++++++++++----------------------- 1 file changed, 23 insertions(+), 29 deletions(-) diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb index 53d5f5e..f616b43 100644 --- a/lib/license/finder/ext/bower.rb +++ b/lib/license/finder/ext/bower.rb @@ -7,9 +7,7 @@ module LicenseFinder end def current_packages - bower_output.map do |bower_module| - map_all(bower_module) - end.flatten + map_all(bower_output).flatten.compact end def possible_package_paths @@ -22,36 +20,32 @@ module LicenseFinder stdout, _stderr, status = Dir.chdir(project_path) do shell.execute([:bower, :list, '--json', '-l', 'action', '--allow-root']) end - return [] unless status.success? + return {} unless status.success? - json = JSON.parse(stdout) - json.fetch('dependencies', {}).values + JSON.parse(stdout) end - def map_all(bower_module) - items = [] - spec = bower_module.fetch('pkgMeta', {}) - - if spec.empty? - endpoint = bower_module.fetch('endpoint', {}) - items << Package.new( - endpoint['name'], - endpoint['target'], - install_path: bower_module['canonicalDir'] - ) - else - items << Package.new( - spec['name'], - spec['version'], - summary: spec['description'], - description: spec['readme'], - homepage: spec['homepage'], - spec_licenses: Package.license_names_from_standard_spec(spec), - install_path: bower_module['canonicalDir'] - ) - end + def map_all(modules) + [map_from(modules)] + + modules.fetch('dependencies', {}).values.map { |x| map_all(x) } + end - items + bower_module.fetch('dependencies', {}).values.map { |x| map_all(x) } + def map_from(bower_module) + meta = bower_module.fetch('pkgMeta', {}) + endpoint = bower_module.fetch('endpoint', {}) + name = meta['name'] || endpoint['name'] + version = meta['version'] || endpoint['target'] + return if name == 'bower' + + Package.new( + name, + version, + summary: meta['description'], + description: meta['readme'], + homepage: meta['homepage'], + spec_licenses: Package.license_names_from_standard_spec(meta), + install_path: bower_module['canonicalDir'] + ) end end end -- cgit v1.2.3 From 74edd0a6d5ddeabad924db114d944424e11aa6f9 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 12:33:21 -0600 Subject: Fix yaml error --- .gitlab/release.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitlab/release.yml b/.gitlab/release.yml index 3bb7d3c..6c8d0d9 100644 --- a/.gitlab/release.yml +++ b/.gitlab/release.yml @@ -9,9 +9,7 @@ major: extends: .release variables: IMAGE_TAG: $MAJOR - trigger: - project: gitlab-org/security-products/analyzers/license-finder - branch: master + trigger: "gitlab-org/security-products/analyzers/license-finder" latest: extends: .release -- cgit v1.2.3 From aa723297a955fae03ca71bb324422590f68ec95b Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 12:37:11 -0600 Subject: Remove trigger --- .gitlab/release.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.gitlab/release.yml b/.gitlab/release.yml index 6c8d0d9..44cc42e 100644 --- a/.gitlab/release.yml +++ b/.gitlab/release.yml @@ -9,7 +9,6 @@ major: extends: .release variables: IMAGE_TAG: $MAJOR - trigger: "gitlab-org/security-products/analyzers/license-finder" latest: extends: .release -- cgit v1.2.3 From acb1635e6edbaea9c620dcdd5714b56cb1594152 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 12:43:11 -0600 Subject: Pass -19 to tar compress program --- config/install.sh | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/config/install.sh b/config/install.sh index 6f684c6..67ce2b2 100644 --- a/config/install.sh +++ b/config/install.sh @@ -160,8 +160,7 @@ rm -fr "$ASDF_DATA_DIR/docs" \ /var/log/* cd /opt -tar -cf /opt/asdf.tar asdf -zstd -19 -k asdf.tar -o asdf.tar.zst +tar --use-compress-program "/usr/bin/zstd -19" -cf /opt/asdf.tar.zst asdf rm -fr /opt/asdf/ cd /usr/lib -- cgit v1.2.3 From 3ff8289b12a8a89730fad979b57fa937db8e5cb5 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 12:46:33 -0600 Subject: Exclude bower package --- lib/license/finder/ext/bower.rb | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb index f616b43..e53e532 100644 --- a/lib/license/finder/ext/bower.rb +++ b/lib/license/finder/ext/bower.rb @@ -7,7 +7,9 @@ module LicenseFinder end def current_packages - map_all(bower_output).flatten.compact + map_all(bower_output).flatten.compact.reject do |package| + package.name == 'bower' + end end def possible_package_paths @@ -33,13 +35,10 @@ module LicenseFinder def map_from(bower_module) meta = bower_module.fetch('pkgMeta', {}) endpoint = bower_module.fetch('endpoint', {}) - name = meta['name'] || endpoint['name'] - version = meta['version'] || endpoint['target'] - return if name == 'bower' Package.new( - name, - version, + meta['name'] || endpoint['name'], + meta['version'] || endpoint['target'], summary: meta['description'], description: meta['readme'], homepage: meta['homepage'], -- cgit v1.2.3 From e2a538ed5c817c792299750ae24c7569b68cd9f3 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 14:00:20 -0600 Subject: Add CHANGELOG entry --- CHANGELOG.md | 5 +++++ Gemfile.lock | 2 +- lib/license/management/version.rb | 2 +- 3 files changed, 7 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f29c22..cbcc4f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # GitLab License management changelog +## v3.9.1 + +- Add `--allow-root` option when install bower packages. (!150) +- Include nested dependencies in scan report for bower projects. (!150) + ## v3.9.0 - Update go list command to be compatible with 1.14 (!143) diff --git a/Gemfile.lock b/Gemfile.lock index e8e1023..4ba5c69 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - license-management (3.9.0) + license-management (3.9.1) license_finder (~> 6.0.0) GEM diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index b5d5bc9..0fe76f1 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.9.0' + VERSION = '3.9.1' end end -- cgit v1.2.3 From 92e06d9fc1303b790398362b9a039e8bba58cd4e Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 14:31:31 -0600 Subject: Fetch bower dependencies from custom repo with custom cert chain --- config/.default-npm-packages | 1 + lib/license/finder/ext/bower.rb | 12 ++++++++---- spec/fixtures/js/bower/bower.json | 4 ++-- spec/integration/js/bower_spec.rb | 26 +++++++++++++++++++++++++- 4 files changed, 36 insertions(+), 7 deletions(-) diff --git a/config/.default-npm-packages b/config/.default-npm-packages index 9991b02..8e16e61 100644 --- a/config/.default-npm-packages +++ b/config/.default-npm-packages @@ -1,3 +1,4 @@ bower +bower-npm-resolver npm-install-peers yarn diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb index e53e532..ccfa7ee 100644 --- a/lib/license/finder/ext/bower.rb +++ b/lib/license/finder/ext/bower.rb @@ -3,13 +3,11 @@ module LicenseFinder class Bower < PackageManager def prepare - shell.execute([:bower, :install, '--allow-root']) + shell.execute([:bower, :install, '--allow-root'], env: default_env) end def current_packages - map_all(bower_output).flatten.compact.reject do |package| - package.name == 'bower' - end + map_all(bower_output).flatten.compact end def possible_package_paths @@ -46,5 +44,11 @@ module LicenseFinder install_path: bower_module['canonicalDir'] ) end + + def default_env + return {} unless shell.custom_certificate_installed? + + { 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.custom_certificate_path.to_s) } + end end end diff --git a/spec/fixtures/js/bower/bower.json b/spec/fixtures/js/bower/bower.json index 4cfbefc..f2760e7 100644 --- a/spec/fixtures/js/bower/bower.json +++ b/spec/fixtures/js/bower/bower.json @@ -1,6 +1,6 @@ { - "name": "bower", - "license": "MIT", + "name": "example-project", + "license": "ISC", "private": true, "ignore": [ "**/.*", diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb index 1f97645..25a45cb 100644 --- a/spec/integration/js/bower_spec.rb +++ b/spec/integration/js/bower_spec.rb @@ -9,9 +9,10 @@ RSpec.describe "bower" do end specify { expect(subject).to match_schema(version: '2.0') } - specify { expect(subject.dependency_names).to match_array(['cli', 'ev-emitter', 'fizzy-ui-utils', 'get-size', 'jquery', 'masonry-layout', 'matches-selector', 'outlayer', 'stimulus.umd']) } + specify { expect(subject.dependency_names).to match_array(['cli', 'ev-emitter', 'example-project', 'fizzy-ui-utils', 'get-size', 'jquery', 'masonry-layout', 'matches-selector', 'outlayer', 'stimulus.umd']) } specify { expect(subject.licenses_for('cli')).to match_array(['Apache-2.0', 'BSD-3-Clause', 'ISC', 'MIT']) } specify { expect(subject.licenses_for('ev-emitter')).to match_array(['MIT']) } + specify { expect(subject.licenses_for('example-project')).to match_array(['ISC']) } specify { expect(subject.licenses_for('fizzy-ui-utils')).to match_array(['MIT']) } specify { expect(subject.licenses_for('get-size')).to match_array(['MIT']) } specify { expect(subject.licenses_for('jquery')).to match_array(['MIT']) } @@ -20,4 +21,27 @@ RSpec.describe "bower" do specify { expect(subject.licenses_for('outlayer')).to match_array(['MIT']) } specify { expect(subject.licenses_for('stimulus.umd')).to match_array(['unknown']) } end + + context "when scanning a bower project with a dependency from a custom registry" do + subject { runner.scan(env: { 'ADDITIONAL_CA_CERT_BUNDLE' => fixture_file_content('js/custom-npm.crt') }) } + + before do + runner.add_file(".npmrc", "registry = https://#{private_npm_host}") + runner.add_file(".bowerrc") do + JSON.pretty_generate({ resolvers: ['bower-npm-resolver'] }) + end + runner.add_file("bower.json") do + JSON.pretty_generate({ + name: "js-bower", + license: "ISC", + dependencies: { lodash: "npm:lodash#4.17.10" } + }) + end + end + + specify { expect(subject).to match_schema(version: '2.0') } + specify { expect(subject.dependency_names).to match_array(%w[js-bower lodash]) } + specify { expect(subject.licenses_for('js-bower')).to match_array(['ISC']) } + specify { expect(subject.licenses_for('lodash')).to match_array(['MIT']) } + end end -- cgit v1.2.3 From d3d899dadde7f0353f4b93a25c40f7b3bec31e8b Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 14:33:09 -0600 Subject: Update CHANGELOG --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cbcc4f7..a973158 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ - Add `--allow-root` option when install bower packages. (!150) - Include nested dependencies in scan report for bower projects. (!150) +- Pass `NPM_CONFIG_CAFILE` to bower install step. (!150) ## v3.9.0 -- cgit v1.2.3 From 69e3a73c81f9dc7c738339d14ed4e7d70579f032 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 14:46:34 -0600 Subject: Update README documentation --- README.md | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 72e6473..d417cdc 100644 --- a/README.md +++ b/README.md @@ -76,13 +76,13 @@ The following table shows which languages and package managers are supported. | Language | Package managers | |------------|-------------------------------------------------------------------| -| JavaScript | [Bower](https://bower.io/), [npm](https://www.npmjs.com/) | -| Go | [Godep](https://github.com/tools/godep), go get | -| Java | [Gradle](https://gradle.org/), [Maven](https://maven.apache.org/) | -| .NET | [Nuget](https://www.nuget.org/) | -| Python | [pip](https://pip.pypa.io/en/stable/) | -| Ruby | [gem](https://rubygems.org/) | -| PHP | [composer](https://getcomposer.org) | +| .NET | [.NET Core CLI][dotnet_core], [Nuget][nuget] | +| Go | [Go modules][gomod], [Godep][godep], go get | +| Java | [Gradle][gradle], [Maven][maven] | +| JavaScript | [npm][npm], [yarn][yarn], [Bower][bower] | +| PHP | [composer][composer] | +| Python | [pip][pip], [pipenv][pipenv] | +| Ruby | [Bundler][bundler] | Inject `SETUP_CMD` to the docker command to override the given package managers and run your custom command to setup your environment with a custom package manager. @@ -150,8 +150,21 @@ If you want to help, read the [contribution guidelines](CONTRIBUTING.md). If an unknown license is detected, please consider updating the mapping defined in [normalized-licenses.yml](https://gitlab.com/gitlab-org/security-products/license-management/blob/master/normalized-licenses.yml). A mapping can be for a detected name or url and must correspond to an SDPX identifier found in [spdx-licenses.json](https://gitlab.com/gitlab-org/security-products/license-management/blob/master/spdx-licenses.json). -[license_finder]: https://rubygems.org/gems/license_finder +[bower]: https://bower.io/ [changelog]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/CHANGELOG.md -[version_rb]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb -[gemspec]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/license-management.gemspec [gemfile_lock]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/Gemfile.lock +[gemspec]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/license-management.gemspec +[license_finder]: https://rubygems.org/gems/license_finder +[npm]: https://www.npmjs.com/ +[version_rb]: https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb +[yarn]: https://yarnpkg.com/ +[gomod]: https://github.com/golang/go/wiki/Modules +[godep]: https://github.com/tools/godep +[gradle]: https://gradle.org/ +[maven]: https://maven.apache.org/ +[nuget]: https://www.nuget.org/ +[dotnet_core]: https://docs.microsoft.com/en-us/dotnet/core/tools/ +[pip]: https://pip.pypa.io/en/stable/ +[pipenv]: https://github.com/pypa/pipenv +[bundler]: https://bundler.io/ +[composer]: https://getcomposer.org -- cgit v1.2.3 From 2bdb3f754f61843fc43d9facce83403b38806266 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 14:54:14 -0600 Subject: Test example bower project --- spec/integration/js/bower_spec.rb | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb index 25a45cb..16a6fed 100644 --- a/spec/integration/js/bower_spec.rb +++ b/spec/integration/js/bower_spec.rb @@ -44,4 +44,19 @@ RSpec.describe "bower" do specify { expect(subject.licenses_for('js-bower')).to match_array(['ISC']) } specify { expect(subject.licenses_for('lodash')).to match_array(['MIT']) } end + + [ + 'https://gitlab.com/gitlab-org/gitter/gitter-marked.git' + ].each do |git_repo| + context "when scanning #{git_repo}" do + subject { runner.scan } + + before do + runner.clone(git_repo) + end + + specify { expect(subject).to match_schema(version: '2.0') } + specify { expect(subject.dependency_names).not_to be_empty } + end + end end -- cgit v1.2.3 From 65ee501b5c82a4f6d332f5ca5048638de10eac11 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 14:57:05 -0600 Subject: Scan example projects --- spec/integration/js/bower_spec.rb | 2 ++ 1 file changed, 2 insertions(+) diff --git a/spec/integration/js/bower_spec.rb b/spec/integration/js/bower_spec.rb index 16a6fed..2127c2e 100644 --- a/spec/integration/js/bower_spec.rb +++ b/spec/integration/js/bower_spec.rb @@ -46,6 +46,8 @@ RSpec.describe "bower" do end [ + 'https://gitlab.com/gitlab-org/ci-training-slides', + 'https://gitlab.com/gitlab-org/frontend/At.js.git', 'https://gitlab.com/gitlab-org/gitter/gitter-marked.git' ].each do |git_repo| context "when scanning #{git_repo}" do -- cgit v1.2.3 From 1ece6417596d751325f0f8a21289d9a85d275f1b Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 15:00:40 -0600 Subject: Reduce compression factor to fit build under 60 minutes --- config/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/install.sh b/config/install.sh index 67ce2b2..7a4b8b9 100644 --- a/config/install.sh +++ b/config/install.sh @@ -160,7 +160,7 @@ rm -fr "$ASDF_DATA_DIR/docs" \ /var/log/* cd /opt -tar --use-compress-program "/usr/bin/zstd -19" -cf /opt/asdf.tar.zst asdf +tar --use-compress-program "/usr/bin/zstd -15" -cf /opt/asdf.tar.zst asdf rm -fr /opt/asdf/ cd /usr/lib -- cgit v1.2.3 From 5c898475335898a7a113fe6681efd3385960710f Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 15:33:11 -0600 Subject: Revert to compression level 19 --- config/install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/install.sh b/config/install.sh index 7a4b8b9..67ce2b2 100644 --- a/config/install.sh +++ b/config/install.sh @@ -160,7 +160,7 @@ rm -fr "$ASDF_DATA_DIR/docs" \ /var/log/* cd /opt -tar --use-compress-program "/usr/bin/zstd -15" -cf /opt/asdf.tar.zst asdf +tar --use-compress-program "/usr/bin/zstd -19" -cf /opt/asdf.tar.zst asdf rm -fr /opt/asdf/ cd /usr/lib -- cgit v1.2.3 From 2d749e4d514b9b5b053879c38ec11ffd83107646 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 19 May 2020 16:16:26 -0600 Subject: Run install steps in parallel where possible --- config/install.sh | 31 +++++++++++++++++-------------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/config/install.sh b/config/install.sh index 67ce2b2..e303910 100644 --- a/config/install.sh +++ b/config/install.sh @@ -77,8 +77,8 @@ apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BF echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list apt-get update -q -apt-get install -y --no-install-recommends dotnet-sdk-3.1 mono-complete -curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe +apt-get install -y --no-install-recommends dotnet-sdk-3.1 mono-complete & +curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe & mkdir -p "$ASDF_DATA_DIR" git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR" @@ -101,7 +101,7 @@ for version in $(asdf list python); do asdf shell python "$version" pip download -d "$HOME/.config/virtualenv/app-data" pip-licenses pip setuptools wheel done - +wait rm -fr /tmp mkdir -p /tmp chmod 777 /tmp @@ -159,24 +159,27 @@ rm -fr "$ASDF_DATA_DIR/docs" \ /var/lib/systemd/* \ /var/log/* +zstd_command="/usr/bin/zstd -19" cd /opt -tar --use-compress-program "/usr/bin/zstd -19" -cf /opt/asdf.tar.zst asdf -rm -fr /opt/asdf/ +tar --use-compress-program "$zstd_command" -cf /opt/asdf.tar.zst asdf & cd /usr/lib -tar --use-compress-program zstd -cf /usr/lib/gcc.tar.zst gcc -rm -fr /usr/lib/gcc +tar --use-compress-program "$zstd_command" -cf /usr/lib/gcc.tar.zst gcc & cd /usr/lib -tar --use-compress-program zstd -cf /usr/lib/mono.tar.zst mono -rm -fr /usr/lib/mono +tar --use-compress-program "$zstd_command" -cf /usr/lib/mono.tar.zst mono & cd /usr/lib -tar --use-compress-program zstd -cf /usr/lib/rustlib.tar.zst rustlib -rm -fr /usr/lib/rustlib +tar --use-compress-program "$zstd_command" -cf /usr/lib/rustlib.tar.zst rustlib & cd /usr/share -tar --use-compress-program zstd -cf /usr/share/dotnet.tar.zst dotnet -rm -fr /usr/share/dotnet - +tar --use-compress-program "$zstd_command" -cf /usr/share/dotnet.tar.zst dotnet & + +wait +rm -fr \ + /opt/asdf/ \ + /usr/lib/gcc \ + /usr/lib/mono \ + /usr/lib/rustlib \ + /usr/share/dotnet echo "Done" -- cgit v1.2.3