From ca3e93e2f67707a016b28e58f8374522a1fb68f0 Mon Sep 17 00:00:00 2001 From: mo khan Date: Tue, 4 Aug 2020 14:51:49 -0600 Subject: Add limited network support for Cargo projects * Add test to scan cargo project with packages from a private registry * Synchronize mono cert store * Wipe nuget package cache before specs * Compress /usr/local/share * Exclude development dependencies * Add CHANGELOG entry --- .gitlab/deb.yml | 1 + .gitlab/test.yml | 6 ++ CHANGELOG.md | 5 ++ Dockerfile | 1 + Gemfile.lock | 2 +- config/files/.bashrc | 1 - config/files/.tool-versions | 2 +- config/install.sh | 8 +- lib/license/finder/ext.rb | 1 + lib/license/finder/ext/cargo.rb | 84 +++++++++++++++++++ lib/license/management/version.rb | 2 +- spec/fixtures/haproxy.cfg | 6 ++ spec/fixtures/rust/cargo/custom-tls/Cargo.lock | 16 ++++ spec/fixtures/rust/cargo/custom-tls/Cargo.toml | 8 ++ spec/fixtures/rust/cargo/custom-tls/src/main.rs | 3 + .../rust/cargo/dev-dependencies/Cargo.lock | 95 ++++++++++++++++++++++ .../rust/cargo/dev-dependencies/Cargo.toml | 11 +++ .../rust/cargo/dev-dependencies/src/main.rs | 3 + spec/fixtures/rust/cargo/hello_world/Cargo.lock | 46 +++++++++++ spec/fixtures/rust/cargo/hello_world/Cargo.toml | 8 ++ spec/fixtures/rust/cargo/hello_world/src/main.rs | 3 + spec/integration/rust/cargo_spec.rb | 58 +++++++++++++ 22 files changed, 362 insertions(+), 8 deletions(-) create mode 100644 lib/license/finder/ext/cargo.rb create mode 100644 spec/fixtures/rust/cargo/custom-tls/Cargo.lock create mode 100644 spec/fixtures/rust/cargo/custom-tls/Cargo.toml create mode 100644 spec/fixtures/rust/cargo/custom-tls/src/main.rs create mode 100644 spec/fixtures/rust/cargo/dev-dependencies/Cargo.lock create mode 100644 spec/fixtures/rust/cargo/dev-dependencies/Cargo.toml create mode 100644 spec/fixtures/rust/cargo/dev-dependencies/src/main.rs create mode 100644 spec/fixtures/rust/cargo/hello_world/Cargo.lock create mode 100644 spec/fixtures/rust/cargo/hello_world/Cargo.toml create mode 100644 spec/fixtures/rust/cargo/hello_world/src/main.rs create mode 100644 spec/integration/rust/cargo_spec.rb diff --git a/.gitlab/deb.yml b/.gitlab/deb.yml index 9a6652d..20b7c93 100644 --- a/.gitlab/deb.yml +++ b/.gitlab/deb.yml @@ -203,3 +203,4 @@ rust-1-45: variables: OMNIBUS_PROJECT: rust RUST_VERSION: '1.45.0' + allow_failure: false diff --git a/.gitlab/test.yml b/.gitlab/test.yml index 6d0dcae..53b1105 100644 --- a/.gitlab/test.yml +++ b/.gitlab/test.yml @@ -105,6 +105,12 @@ integration-ruby: RSPEC_DIR: spec/integration/ruby needs: ['build-docker-image'] +integration-rust: + extends: .rspec + variables: + RSPEC_DIR: spec/integration/rust + needs: ['build-docker-image'] + .functional: stage: test trigger: gitlab-org/security-products/tests/common diff --git a/CHANGELOG.md b/CHANGELOG.md index 05f56bf..6fb5fdb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # GitLab License management changelog +## v3.21.0 + +- Provide limited network connectivity for [cargo](https://doc.rust-lang.org/cargo/) projects (!201) +- Exclude development dependencies from [cargo](https://doc.rust-lang.org/cargo/) project scans (!201) + ## v3.20.1 - Ensure that error messages are converted to strings before writing to the log. (!203) diff --git a/Dockerfile b/Dockerfile index dab3bff..ac5083f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,7 @@ COPY config/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc RUN mkdir -p /opt/toolcache COPY --from=mono-builder /opt/toolcache/*.deb /opt/toolcache/mono/ COPY pkg/license*.deb /opt/toolcache/ +COPY pkg/rust*.deb /opt/toolcache/ COPY config/install.sh /opt/install.sh RUN bash /opt/install.sh COPY run.sh / diff --git a/Gemfile.lock b/Gemfile.lock index 7356717..f37952b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -8,7 +8,7 @@ GIT PATH remote: . specs: - license-management (3.20.1) + license-management (3.21.0) license_finder (~> 6.6.0) GEM diff --git a/config/files/.bashrc b/config/files/.bashrc index fa70e9c..b37ae94 100644 --- a/config/files/.bashrc +++ b/config/files/.bashrc @@ -70,7 +70,6 @@ inflate /usr/lib/erlang.tar.zst /usr/lib inflate /usr/lib/gcc.tar.zst /usr/lib inflate /usr/lib/git-core.tar.zst /usr/lib inflate /usr/lib/llvm-7.tar.zst /usr/lib -inflate /usr/lib/rustlib.tar.zst /usr/lib inflate /usr/share.tar.zst /usr # shellcheck source=/dev/null diff --git a/config/files/.tool-versions b/config/files/.tool-versions index 41b41e6..968bd25 100644 --- a/config/files/.tool-versions +++ b/config/files/.tool-versions @@ -7,5 +7,5 @@ nodejs 12.16.3 10.20.1 php 7.4.5 python 3.8.2 2.7.18 ruby 2.6.6 -rust system +rust 1.45.0 sbt 1.3.8 diff --git a/config/install.sh b/config/install.sh index 62a64fd..dc32ed2 100644 --- a/config/install.sh +++ b/config/install.sh @@ -62,7 +62,6 @@ apt-get install -y --no-install-recommends \ python-openssl \ re2c \ rebar \ - rustc \ software-properties-common \ sudo \ tk-dev \ @@ -94,6 +93,7 @@ while IFS= read -r line; do done < "/opt/gitlab/.tool-versions" bash "$ASDF_DATA_DIR/plugins/nodejs/bin/import-release-team-keyring" asdf install +rm -fr "$ASDF_DATA_DIR/installs/rust" asdf reshim asdf current @@ -198,11 +198,11 @@ tar --use-compress-program "$zstd_command" -cf /usr/lib/erlang.tar.zst erlang & tar --use-compress-program "$zstd_command" -cf /usr/lib/gcc.tar.zst gcc & tar --use-compress-program "$zstd_command" -cf /usr/lib/git-core.tar.zst git-core & tar --use-compress-program "$zstd_command" -cf /usr/lib/llvm-7.tar.zst llvm-7 & -tar --use-compress-program "$zstd_command" -cf /usr/lib/rustlib.tar.zst rustlib & wait +# shellcheck disable=SC2114 rm -fr \ - /opt/asdf/ \ + "$ASDF_DATA_DIR" \ /opt/gitlab/.m2 \ /opt/gitlab/.cache \ /opt/gitlab/.config \ @@ -213,5 +213,5 @@ rm -fr \ /usr/lib/gcc \ /usr/lib/git-core \ /usr/lib/llvm-7 \ - /usr/lib/rustlib + /usr/share echo -e "section_end:$(date +%s):compress_files\r\e[0K" diff --git a/lib/license/finder/ext.rb b/lib/license/finder/ext.rb index e524771..fb593cc 100644 --- a/lib/license/finder/ext.rb +++ b/lib/license/finder/ext.rb @@ -2,6 +2,7 @@ require 'license/finder/ext/bower' require 'license/finder/ext/bundler' +require 'license/finder/ext/cargo' require 'license/finder/ext/composer' require 'license/finder/ext/conan' require 'license/finder/ext/dependency' diff --git a/lib/license/finder/ext/cargo.rb b/lib/license/finder/ext/cargo.rb new file mode 100644 index 0000000..20b1b03 --- /dev/null +++ b/lib/license/finder/ext/cargo.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +module LicenseFinder + class Cargo < PackageManager + SENTINEL = ' -|- ' + + def installed?(*) + true + end + + def prepare + create_vendor_path + + within_project_path do + shell.execute([:dpkg, '-i', '/opt/toolcache/rust*.deb']) + shell.execute([:asdf, :reshim]) + shell.execute([:env, '|', :sort, '&&', :asdf, :exec, :cargo, :fetch, '-vv'], env: default_env) + end + end + + def current_packages + within_project_path do + stdout, _stderr, status = shell.execute(scan_command) + status.success? ? parse(stdout) : [] + end + end + + def possible_package_paths + [ + project_path.join('Cargo.lock'), + project_path.join('Cargo.toml') + ] + end + + private + + def parse(stdout) + stdout.each_line.map do |line| + next if line.include?(project_path.to_s) + + map_from(line.chomp) + end.compact + end + + def scan_command + [ + :asdf, :exec, + :cargo, :tree, '--edges=no-dev', + "--color=never", + "--offline", + "--prefix=none", + "-f='{p}#{SENTINEL}{l}'" + ] + end + + def default_env + @default_env ||= { + 'CARGO_HOME' => vendor_path.to_s, + 'CARGO_TARGET_DIR' => vendor_path.to_s, + 'HTTP_TIMEOUT' => '10', + 'HTTP_CAINFO' => shell.default_certificate_path.to_s, + 'TERM' => 'dumb' + } + end + + def map_from(line) + name_version, license = line.split(SENTINEL, 2) + name, version = name_version.split(' ', 2) + + Dependency.new( + 'Cargo', + name, + version, + detection_path: detected_package_path, + install_path: path_to(name, version), + spec_licenses: license ? license.split('/') : [] + ) + end + + def path_to(name, version) + Dir.glob("/opt/asdf/installs/rust/**/registry/src/**/#{name}-#{version}")[0] + end + end +end diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index 9e5f556..e3de501 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.20.1' + VERSION = '3.21.0' end end diff --git a/spec/fixtures/haproxy.cfg b/spec/fixtures/haproxy.cfg index 4a5b982..d5b2647 100644 --- a/spec/fixtures/haproxy.cfg +++ b/spec/fixtures/haproxy.cfg @@ -20,6 +20,7 @@ defaults frontend www-https bind *:443 ssl crt wildcard.test.pem + acl cargo-backend ssl_fc_sni cargo.test acl composer-backend ssl_fc_sni composer.test acl goproxy-backend ssl_fc_sni goproxy.test acl maven-backend ssl_fc_sni maven.test @@ -29,6 +30,7 @@ frontend www-https acl rubygems-backend ssl_fc_sni rubygems.test http-request replace-header Host .* api.nuget.org if nuget-backend + http-request replace-header Host .* github.com if cargo-backend http-request replace-header Host .* packagist.org if composer-backend http-request replace-header Host .* proxy.golang.org if goproxy-backend http-request replace-header Host .* pypi.org if pypi-backend @@ -36,6 +38,7 @@ frontend www-https http-request replace-header Host .* repo1.maven.org if maven-backend http-request replace-header Host .* rubygems.org if rubygems-backend + use_backend cargo-backend if cargo-backend use_backend composer-backend if composer-backend use_backend goproxy-backend if goproxy-backend use_backend maven-backend if maven-backend @@ -44,6 +47,9 @@ frontend www-https use_backend pypi-backend if pypi-backend use_backend rubygems-backend if rubygems-backend +backend cargo-backend + server www1 github.com:443 ssl verify none + backend composer-backend server www1 packagist.org:443 ssl verify none diff --git a/spec/fixtures/rust/cargo/custom-tls/Cargo.lock b/spec/fixtures/rust/cargo/custom-tls/Cargo.lock new file mode 100644 index 0000000..52702bd --- /dev/null +++ b/spec/fixtures/rust/cargo/custom-tls/Cargo.lock @@ -0,0 +1,16 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "custom-tls" +version = "0.1.0" +dependencies = [ + "libc 0.2.74 (registry+https://cargo.test/rust-lang/crates.io-index)", +] + +[[package]] +name = "libc" +version = "0.2.74" +source = "registry+https://cargo.test/rust-lang/crates.io-index" + +[metadata] +"checksum libc 0.2.74 (registry+https://cargo.test/rust-lang/crates.io-index)" = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" diff --git a/spec/fixtures/rust/cargo/custom-tls/Cargo.toml b/spec/fixtures/rust/cargo/custom-tls/Cargo.toml new file mode 100644 index 0000000..5058806 --- /dev/null +++ b/spec/fixtures/rust/cargo/custom-tls/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "custom-tls" +version = "0.1.0" +authors = ["example "] +edition = "2018" + +[dependencies] +libc = "0.2.74" diff --git a/spec/fixtures/rust/cargo/custom-tls/src/main.rs b/spec/fixtures/rust/cargo/custom-tls/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/spec/fixtures/rust/cargo/custom-tls/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/spec/fixtures/rust/cargo/dev-dependencies/Cargo.lock b/spec/fixtures/rust/cargo/dev-dependencies/Cargo.lock new file mode 100644 index 0000000..5c2a516 --- /dev/null +++ b/spec/fixtures/rust/cargo/dev-dependencies/Cargo.lock @@ -0,0 +1,95 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "ansi_term" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "dev-dependencies" +version = "0.1.0" +dependencies = [ + "diesel 1.4.5 (registry+https://github.com/rust-lang/crates.io-index)", + "pretty_assertions 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "diesel" +version = "1.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)", + "diesel_derives 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "diesel_derives" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "syn 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "difference" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] +name = "pretty_assertions" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)", + "difference 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "proc-macro2" +version = "1.0.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "quote" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "syn" +version = "1.0.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)", + "quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)", + "unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "unicode-xid" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[metadata] +"checksum ansi_term 0.9.0 (registry+https://github.com/rust-lang/crates.io-index)" = "23ac7c30002a5accbf7e8987d0632fa6de155b7c3d39d0067317a391e00a2ef6" +"checksum byteorder 1.3.4 (registry+https://github.com/rust-lang/crates.io-index)" = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" +"checksum diesel 1.4.5 (registry+https://github.com/rust-lang/crates.io-index)" = "3e2de9deab977a153492a1468d1b1c0662c1cf39e5ea87d0c060ecd59ef18d8c" +"checksum diesel_derives 1.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "45f5098f628d02a7a0f68ddba586fb61e80edec3bdc1be3b921f4ceec60858d3" +"checksum difference 1.0.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b3304d19798a8e067e48d8e69b2c37f0b5e9b4e462504ad9e27e9f3fce02bba8" +"checksum pretty_assertions 0.4.1 (registry+https://github.com/rust-lang/crates.io-index)" = "28ea5118e2f41bfbc974b28d88c07621befd1fa5d6ec23549be96302a1a59dd2" +"checksum proc-macro2 1.0.19 (registry+https://github.com/rust-lang/crates.io-index)" = "04f5f085b5d71e2188cb8271e5da0161ad52c3f227a661a3c135fdf28e258b12" +"checksum quote 1.0.7 (registry+https://github.com/rust-lang/crates.io-index)" = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +"checksum syn 1.0.38 (registry+https://github.com/rust-lang/crates.io-index)" = "e69abc24912995b3038597a7a593be5053eb0fb44f3cc5beec0deb421790c1f4" +"checksum unicode-xid 0.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" diff --git a/spec/fixtures/rust/cargo/dev-dependencies/Cargo.toml b/spec/fixtures/rust/cargo/dev-dependencies/Cargo.toml new file mode 100644 index 0000000..7adf447 --- /dev/null +++ b/spec/fixtures/rust/cargo/dev-dependencies/Cargo.toml @@ -0,0 +1,11 @@ +[package] +name = "dev-dependencies" +version = "0.1.0" +authors = ["example "] +edition = "2018" + +[dependencies] +diesel = "1.4.5" + +[dev-dependencies] +pretty_assertions = "0.4.0" diff --git a/spec/fixtures/rust/cargo/dev-dependencies/src/main.rs b/spec/fixtures/rust/cargo/dev-dependencies/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/spec/fixtures/rust/cargo/dev-dependencies/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/spec/fixtures/rust/cargo/hello_world/Cargo.lock b/spec/fixtures/rust/cargo/hello_world/Cargo.lock new file mode 100644 index 0000000..0d0a86c --- /dev/null +++ b/spec/fixtures/rust/cargo/hello_world/Cargo.lock @@ -0,0 +1,46 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "hello_world" +version = "0.1.0" +dependencies = [ + "time", +] + +[[package]] +name = "libc" +version = "0.2.74" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2f02823cf78b754822df5f7f268fb59822e7296276d3e069d8e8cb26a14bd10" + +[[package]] +name = "time" +version = "0.1.43" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca8a50ef2360fbd1eeb0ecd46795a87a19024eb4b53c5dc916ca1fd95fe62438" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" diff --git a/spec/fixtures/rust/cargo/hello_world/Cargo.toml b/spec/fixtures/rust/cargo/hello_world/Cargo.toml new file mode 100644 index 0000000..8331179 --- /dev/null +++ b/spec/fixtures/rust/cargo/hello_world/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "hello_world" +version = "0.1.0" +authors = ["example "] +edition = "2018" + +[dependencies] +time = "0.1.12" diff --git a/spec/fixtures/rust/cargo/hello_world/src/main.rs b/spec/fixtures/rust/cargo/hello_world/src/main.rs new file mode 100644 index 0000000..e7a11a9 --- /dev/null +++ b/spec/fixtures/rust/cargo/hello_world/src/main.rs @@ -0,0 +1,3 @@ +fn main() { + println!("Hello, world!"); +} diff --git a/spec/integration/rust/cargo_spec.rb b/spec/integration/rust/cargo_spec.rb new file mode 100644 index 0000000..77ad447 --- /dev/null +++ b/spec/integration/rust/cargo_spec.rb @@ -0,0 +1,58 @@ +require 'spec_helper' + +RSpec.describe "cargo" do + subject { runner.scan(env: env) } + + let(:env) { {} } + + context "when scanning a cargo project" do + before do + runner.mount(dir: fixture_file('rust/cargo/hello_world')) + end + + specify { expect(subject).to match_schema } + + specify do + expect(subject.dependency_names).to match_array(%w[libc time]) + end + + specify { expect(subject.licenses_for('libc')).to match_array(['mit or apache-2.0']) } + specify { expect(subject.licenses_for('time')).to match_array(['Apache-2.0', 'MIT']) } + end + + context "when fetching dependencies from a custom registry" do + before do + add_host('cargo.test', '127.0.0.1') + start_proxy_server + runner.mount(dir: fixture_file('rust/cargo/custom-tls')) + end + + context "when the CA certificate is provided" do + let(:env) { { 'ADDITIONAL_CA_CERT_BUNDLE' => x509_certificate('wildcard.test').read } } + + specify do + expect(subject).to match_schema + expect(subject.dependency_names).to match_array(['libc']) + expect(subject.licenses_for('libc')).to match_array(['mit or apache-2.0']) + end + end + + context "when the CA certificate is NOT provided" do + let(:env) { {} } + + specify { expect(subject).to match_schema } + end + end + + context "when scanning a project with dev dependencies" do + before do + runner.mount(dir: fixture_file('rust/cargo/dev-dependencies')) + end + + it 'excludes them from the report' do + expect(subject).to match_schema + expect(subject.dependency_names).to include('diesel') + expect(subject.dependency_names).not_to include('pretty_assertions') + end + end +end -- cgit v1.2.3