diff options
| -rw-r--r-- | .gitlab-ci.yml | 19 | ||||
| -rw-r--r-- | CHANGELOG.md | 6 | ||||
| -rw-r--r-- | Dockerfile | 38 | ||||
| -rw-r--r-- | Gemfile.lock | 16 | ||||
| -rw-r--r-- | README.md | 2 | ||||
| -rwxr-xr-x | bin/test-local | 15 | ||||
| -rw-r--r-- | license-management.gemspec | 2 | ||||
| -rwxr-xr-x | run.sh | 16 | ||||
| -rw-r--r-- | spec/integration/python/pip_spec.rb | 47 | ||||
| -rw-r--r-- | spec/integration/ruby/bundler_spec.rb | 88 | ||||
| -rw-r--r-- | spec/spec_helper.rb | 12 | ||||
| -rw-r--r-- | spec/support/integration_test_helper.rb | 37 | ||||
| -rw-r--r-- | spec/unit/license/management/report/v2_spec.rb (renamed from spec/license/management/report/v2_spec.rb) | 0 | ||||
| -rw-r--r-- | spec/unit/license/management/repository_spec.rb (renamed from spec/license/management/repository_spec.rb) | 0 |
14 files changed, 262 insertions, 36 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 8347a2a..02c188c 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -48,12 +48,29 @@ container_scanning: - results/ when: always +size: + image: docker:stable + stage: test + allow_failure: true # temporary until we can shrink the image size. + timeout: 1 minute + script: + - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY + - docker pull $TMP_IMAGE + unit: stage: test script: - gem install bundler -v '~> 2.0' - bundle install - - bundle exec rspec + - bundle exec rspec spec/unit + +integration: + image: + name: $TMP_IMAGE + entrypoint: [""] + stage: test + script: + - bash -lc "cd $LM_HOME && bundle install && bundle exec rspec spec/integration" QA:php-composer: extends: .QA diff --git a/CHANGELOG.md b/CHANGELOG.md index 507f449..4e82432 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,12 @@ # GitLab License management changelog +## v2.3.1 + +- Run gradle without tests by default. (!102) + ## v2.3.0 -- Run gradle without tests by default. +- Install Python 3.8.1 as the default python (!101) ## v2.2.3 @@ -1,12 +1,8 @@ -ARG LICENSE_FINDER_VERSION=5.11.1 - -FROM licensefinder/license_finder:$LICENSE_FINDER_VERSION -MAINTAINER GitLab - -ARG LICENSE_FINDER_VERSION -ENV LICENSE_FINDER_VERSION $LICENSE_FINDER_VERSION - -# Install JDK 11 +FROM licensefinder/license_finder:5.11.1 +ENV PATH="${PATH}:/root/.asdf/shims:/root/.asdf/bin" +ENV LM_HOME=/opt/license-management +ENV LM_PYTHON_VERSION 3 +ENV LM_REPORT_VERSION ${LM_REPORT_VERSION:-2} RUN cd /tmp && \ wget --quiet --no-cookies https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.5%2B10/OpenJDK11U-jdk_x64_linux_hotspot_11.0.5_10.tar.gz -O jdk-11.tgz && \ tar xf /tmp/jdk-11.tgz && \ @@ -17,22 +13,24 @@ RUN npm install npm-install-peers # Don't let Rubygem fail with the numerous projects using PG or MySQL, # install realpath, includes for python3, and pip for python3 # Install additional php packages for better composer package support -# Install .NET Core 2.2, 3.0 because it is not installed in the license_finder image (https://github.com/pivotal/LicenseFinder/pull/632). RUN add-apt-repository ppa:ondrej/php -y && apt-get update -y && \ apt-get install -y --no-install-recommends \ - libpq-dev libmysqlclient-dev realpath python3-dev python3-pip dotnet-sdk-2.2 dotnet-sdk-3.0 \ + bsdmainutils \ + libjpeg8-dev \ + zlib1g-dev \ + libpq-dev libmysqlclient-dev realpath dotnet-sdk-2.2 dotnet-sdk-3.0 \ php7.1-mbstring php7.1-intl php7.1-xml php7.1-soap -y && \ + git clone --depth 1 --branch v0.7.6 https://github.com/asdf-vm/asdf.git $HOME/.asdf && \ + echo 'pip' >> $HOME/.default-python-packages && \ + echo 'setuptools' >> $HOME/.default-python-packages && \ + echo '\n. $HOME/.asdf/asdf.sh' >> $HOME/.bashrc && \ + asdf plugin-add python && \ + echo 'python 3.8.1 3.5.9 2.7.17' >> $HOME/.tool-versions && \ + asdf install && \ + asdf global python 3.8.1 && \ + asdf reshim && \ rm -rf /var/lib/apt/lists/* -# Install setuptools, and fetch a recent version pip to be installed later on -RUN pip install --upgrade pip setuptools \ - && python3 -m pip install --upgrade pip setuptools - -# Version of Python, defaults to Python 3.5 -ARG LM_PYTHON_VERSION=3.5 -ENV LM_PYTHON_VERSION $LM_PYTHON_VERSION -ENV LM_REPORT_VERSION ${LM_REPORT_VERSION:-1} - COPY test /test COPY run.sh / COPY . /opt/license-management/ diff --git a/Gemfile.lock b/Gemfile.lock index 58ab499..7d503e8 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,8 @@ PATH remote: . specs: - license-management (2.2.3) - license_finder + license-management (2.3.0) + license_finder (~> 5.11) GEM remote: https://rubygems.org/ @@ -20,17 +20,17 @@ GEM rspec-core (~> 3.9.0) rspec-expectations (~> 3.9.0) rspec-mocks (~> 3.9.0) - rspec-core (3.9.0) - rspec-support (~> 3.9.0) + rspec-core (3.9.1) + rspec-support (~> 3.9.1) rspec-expectations (3.9.0) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) - rspec-mocks (3.9.0) + rspec-mocks (3.9.1) diff-lcs (>= 1.2.0, < 2.0) rspec-support (~> 3.9.0) - rspec-support (3.9.0) + rspec-support (3.9.2) rubyzip (2.0.0) - thor (0.20.3) + thor (1.0.1) toml (0.2.0) parslet (~> 1.8.0) with_env (1.1.0) @@ -44,4 +44,4 @@ DEPENDENCIES rspec (~> 3.9) BUNDLED WITH - 2.0.2 + 2.1.2 @@ -120,7 +120,7 @@ to the docker command. 2. Create a new `.env.*` that corresponds to the version to release and specify the default configuration. ```text - LM_PYTHON_VERSION '3.5' + LM_PYTHON_VERSION '3' LM_REPORT_VERSION '2.0' ``` diff --git a/bin/test-local b/bin/test-local new file mode 100755 index 0000000..9a84b40 --- /dev/null +++ b/bin/test-local @@ -0,0 +1,15 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")/.." + +PROJECT_PATH="$1" + +if [ -z "$LM_HOME" ]; then + export IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest} + + docker run --rm --volume "$PROJECT_PATH":/code "$IMAGE_NAME" analyze /code +else + bash -l ./run.sh analyze "$PROJECT_PATH" +fi diff --git a/license-management.gemspec b/license-management.gemspec index 6076ebf..c58bbdc 100644 --- a/license-management.gemspec +++ b/license-management.gemspec @@ -27,6 +27,6 @@ Gem::Specification.new do |spec| spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } spec.require_paths = ['lib'] - spec.add_dependency 'license_finder', ENV['LICENSE_FINDER_VERSION'] + spec.add_dependency 'license_finder', '~> 5.11' spec.add_development_dependency 'rspec', '~> 3.9' end @@ -54,7 +54,6 @@ if [ "$COMMAND" = "/test/test.sh" ] ; then fi # Check number of arguments - if [ "$COMMAND" = "analyze" -a $# -ne 1 ] ; then echo "$usage" exit 1 @@ -65,17 +64,27 @@ if [ "$COMMAND" = "test" -a $# -ne 3 ] ; then exit 1 fi +source $HOME/.asdf/asdf.sh + # Switch to Python 3 if requesting case "$LM_PYTHON_VERSION" in "2"|"2.7") echo "using python $LM_PYTHON_VERSION" LICENSE_FINDER_CLI_OPTS="--python-version 2 $LICENSE_FINDER_CLI_OPTS" - pip2 --version + asdf shell python "$(asdf list python | awk '{ print $1 }' | grep '^2')" + pip --version + ;; + + "3") + echo "switching to python $LM_PYTHON_VERSION" + LICENSE_FINDER_CLI_OPTS="--python-version 3 $LICENSE_FINDER_CLI_OPTS" + pip --version ;; - "3"|"3.5") + "3.5") echo "switching to python $LM_PYTHON_VERSION" LICENSE_FINDER_CLI_OPTS="--python-version 3 $LICENSE_FINDER_CLI_OPTS" + asdf shell python 3.5.9 pip --version ;; @@ -114,7 +123,6 @@ case "$COMMAND" in # rvm pulls outdated gems # need this to update system bundler # We need to install the license_finder gem into this Ruby version too. - gem install license_finder -v "$LICENSE_FINDER_VERSION" gem install --no-document /opt/license-management/*.gem fi diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb new file mode 100644 index 0000000..5ff5f60 --- /dev/null +++ b/spec/integration/python/pip_spec.rb @@ -0,0 +1,47 @@ +require 'spec_helper' + +RSpec.describe "pip" do + context "when a project depends on the latest version of pip" do + let(:requirements) { "sentry-sdk>=0.7.7" } + + it 'produces a valid report' do + runner.add_file('requirements.txt', requirements) + + report = runner.scan + + expect(report).not_to be_empty + expect(report[:version]).to start_with('2') + expect(report[:dependencies].map { |x| x[:name] }).to include("sentry-sdk") + expect(report[:dependencies].find { |x| x[:name] == 'sentry-sdk' }[:licenses]).to match_array(["BSD-4-Clause"]) + end + end + + context "when the project has a dependency that depends on a minimum of python 3.6" do + let(:requirements) do + [ + 'boto3', + 'aws-lambda-context>=1.0.0', + 'jsonschema>=3.0.0', + 'python-json-logger>=0.1.10', + 'sentry-sdk>=0.7.7', + 'https://s3-eu-west-1.amazonaws.com/new10-pypi/new10-logging-1.1.4.tar.gz', + 'ptvsd', + 'pylint', + 'flake8', + 'bandit', + 'pydocstyle' + ].join("\n") + end + + it 'produces a valid report' do + runner.add_file('requirements.txt', requirements) + + report = runner.scan + + expect(report).not_to be_empty + expect(report[:version]).to start_with('2') + expect(report[:licenses]).not_to be_empty + expect(report[:dependencies]).not_to be_empty + end + end +end diff --git a/spec/integration/ruby/bundler_spec.rb b/spec/integration/ruby/bundler_spec.rb new file mode 100644 index 0000000..179da2a --- /dev/null +++ b/spec/integration/ruby/bundler_spec.rb @@ -0,0 +1,88 @@ +require 'spec_helper' + +RSpec.describe "bundler" do + context "when the project depends on an older version of ruby specified in a `.ruby-version` file" do + it 'installs the required ruby and produces a valid report' do + runner.add_file('.ruby-version', 'ruby-2.4.9') + runner.add_file('Gemfile') do + <<~RAW +source 'https://rubygems.org' + +gem 'saml-kit' + RAW + end + + report = runner.scan + expect(report).not_to be_empty + expect(report[:licenses]).not_to be_empty + expect(report[:dependencies].map { |x| x[:name] }).to include("saml-kit") + end + end + + context "when a project depends on an older version of bundler" do + it 'produces a valid report' do + runner.add_file('Gemfile') do + <<~RAW +source 'https://rubygems.org' + +gem 'saml-kit' + RAW + end + runner.add_file('Gemfile.lock') do + <<~RAW +GEM + remote: https://rubygems.org/ + specs: + activemodel (6.0.2.1) + activesupport (= 6.0.2.1) + activesupport (6.0.2.1) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + zeitwerk (~> 2.2) + builder (3.2.4) + concurrent-ruby (1.1.5) + i18n (1.7.1) + concurrent-ruby (~> 1.0) + mini_portile2 (2.4.0) + minitest (5.13.0) + net-hippie (0.2.7) + nokogiri (1.10.7) + mini_portile2 (~> 2.4.0) + saml-kit (1.1.0) + activemodel (>= 4.2.0) + net-hippie (~> 0.1) + xml-kit (>= 0.3.0, < 1.0.0) + thread_safe (0.3.6) + tilt (2.0.10) + tzinfo (1.2.6) + thread_safe (~> 0.1) + xml-kit (0.4.0) + activemodel (>= 4.2.0) + builder (~> 3.2) + nokogiri (~> 1.10) + tilt (>= 1.4.1) + xmldsig (~> 0.6) + xmldsig (0.6.6) + nokogiri (>= 1.6.8, < 2.0.0) + zeitwerk (2.2.2) + +PLATFORMS + ruby + +DEPENDENCIES + saml-kit + +BUNDLED WITH + 1.17.3 + RAW + end + + report = runner.scan + expect(report).not_to be_empty + expect(report[:licenses]).not_to be_empty + expect(report[:dependencies].map { |x| x[:name] }).to include("saml-kit") + end + end +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index 2282613..be7673c 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -1,6 +1,18 @@ require 'license/management' +require 'json' +require 'support/integration_test_helper' RSpec.configure do |config| + config.include IntegrationTestHelper, type: :integration + config.define_derived_metadata(file_path: /\/spec\/integration/) do |metadata| + metadata[:type] = :integration + end + config.before(:suite) do + system('./bin/docker-build') unless ENV['LM_HOME'] + end + config.after(:example, type: :integration) do + runner.cleanup + end config.expect_with :rspec do |expectations| expectations.include_chain_clauses_in_custom_matcher_descriptions = true end diff --git a/spec/support/integration_test_helper.rb b/spec/support/integration_test_helper.rb new file mode 100644 index 0000000..df75176 --- /dev/null +++ b/spec/support/integration_test_helper.rb @@ -0,0 +1,37 @@ +module IntegrationTestHelper + class IntegrationTestRunner + attr_reader :project_path + + def initialize(project_path = Dir.mktmpdir('lm')) + @project_path = project_path + end + + def add_file(name, content = nil) + full_path = "#{project_path}/#{name}" + IO.write(full_path, block_given? ? yield : content) + end + + def scan(env: {}) + return {} unless execute(env, './bin/test-local', project_path) + + report_path = "#{project_path}/gl-license-management-report.json" + return {} unless File.exist?(report_path) + + JSON.parse(IO.read(report_path), symbolize_names: true) + end + + def execute(env = {}, *args) + Bundler.with_clean_env do + system(env, *args) + end + end + + def cleanup + FileUtils.rm_rf(project_path) if Dir.exist?(project_path) + end + end + + def runner(*args) + @runner ||= IntegrationTestRunner.new(*args) + end +end diff --git a/spec/license/management/report/v2_spec.rb b/spec/unit/license/management/report/v2_spec.rb index 4da973c..4da973c 100644 --- a/spec/license/management/report/v2_spec.rb +++ b/spec/unit/license/management/report/v2_spec.rb diff --git a/spec/license/management/repository_spec.rb b/spec/unit/license/management/repository_spec.rb index 6ebc09e..6ebc09e 100644 --- a/spec/license/management/repository_spec.rb +++ b/spec/unit/license/management/repository_spec.rb |