summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authormo khan <mo.khan@gmail.com>2020-07-13 09:57:29 -0600
committermo khan <mo.khan@gmail.com>2020-08-21 16:31:57 -0600
commitcbc4d1e328fd0e578f038d57fbd6b9354dafa79e (patch)
tree99b53a90cd7c0996a2d0993b41eb56a695bcc996
parenteb11eeb1018c4d32e8dbaf3f734e588205f0671f (diff)
Install tools from .deb package
-rw-r--r--.gitlab-ci.yml2
-rw-r--r--.gitlab/deb.yml58
-rw-r--r--CHANGELOG.md4
-rw-r--r--Dockerfile10
-rw-r--r--Gemfile2
-rw-r--r--Gemfile.lock2
-rwxr-xr-xbin/docker-omnibus10
-rwxr-xr-xbin/omnibus17
-rw-r--r--config/files/.bashrc8
-rw-r--r--config/files/.default-python-packages2
-rw-r--r--config/files/.tool-versions12
-rw-r--r--config/install.sh74
-rwxr-xr-xconfig/scripts/asdf/postinst3
-rwxr-xr-xconfig/scripts/license_management/postinst11
-rwxr-xr-xconfig/scripts/license_management/postrm3
-rw-r--r--config/software/asdf.rb2
-rw-r--r--config/software/asdf_golang.rb6
-rw-r--r--config/software/asdf_python.rb44
-rw-r--r--config/software/license_management.rb4
-rw-r--r--lib/license/finder/ext/bower.rb45
-rw-r--r--lib/license/finder/ext/bundler.rb41
-rw-r--r--lib/license/finder/ext/cargo.rb6
-rw-r--r--lib/license/finder/ext/composer.rb1
-rw-r--r--lib/license/finder/ext/conan.rb10
-rw-r--r--lib/license/finder/ext/dotnet.rb14
-rw-r--r--lib/license/finder/ext/go_modules.rb5
-rw-r--r--lib/license/finder/ext/gradle.rb25
-rw-r--r--lib/license/finder/ext/maven.rb29
-rw-r--r--lib/license/finder/ext/npm.rb34
-rw-r--r--lib/license/finder/ext/nuget.rb16
-rw-r--r--lib/license/finder/ext/package_manager.rb19
-rw-r--r--lib/license/finder/ext/pip.rb90
-rw-r--r--lib/license/finder/ext/pipenv.rb53
-rw-r--r--lib/license/finder/ext/yarn.rb31
-rw-r--r--lib/license/management.rb2
-rw-r--r--lib/license/management/python.rb61
-rw-r--r--lib/license/management/shell.rb52
-rw-r--r--lib/license/management/tool_box.rb123
-rw-r--r--lib/license/management/version.rb2
-rw-r--r--omnibus.rb2
-rwxr-xr-xrun.sh43
-rw-r--r--spec/fixtures/python/pipenv/specific-python-version/.tool-versions1
-rw-r--r--spec/integration/dotnet/nuget_spec.rb1
-rw-r--r--spec/integration/java/gradle_spec.rb6
-rw-r--r--spec/integration/java/maven_spec.rb42
-rw-r--r--spec/integration/python/pip_spec.rb59
-rw-r--r--spec/integration/python/pipenv_spec.rb4
-rw-r--r--spec/integration/ruby/bundler_spec.rb17
-rw-r--r--spec/support/shared.rb8
-rw-r--r--spec/unit/license_finder/bundler_spec.rb58
-rw-r--r--spec/unit/license_finder/dotnet_spec.rb37
-rw-r--r--spec/unit/license_finder/gradle_spec.rb65
-rw-r--r--spec/unit/license_finder/maven_spec.rb65
-rw-r--r--spec/unit/license_finder/pip_spec.rb69
-rw-r--r--spec/unit/license_finder/pipenv_spec.rb75
-rw-r--r--spec/unit/license_finder/yarn_spec.rb33
56 files changed, 1002 insertions, 516 deletions
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1b25016..6c7a8b7 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -34,6 +34,7 @@ default:
container_scanning:
variables:
CI_APPLICATION_REPOSITORY: $CI_REGISTRY_IMAGE/license-finder # only predefined variables are parameter-expanded (no $TMP_IMAGE)
+ needs: ['build-docker-image']
code_quality:
before_script:
@@ -46,6 +47,7 @@ dependency_scanning:
services:
- docker:stable-dind
tags: [gitlab-org-docker]
+ needs: []
license_scanning:
image:
diff --git a/.gitlab/deb.yml b/.gitlab/deb.yml
index f4f3bfe..a0b5f29 100644
--- a/.gitlab/deb.yml
+++ b/.gitlab/deb.yml
@@ -12,23 +12,23 @@
paths:
- tmp/omnibus
-asdf:
+asdf-0.7.8:
extends: .deb
variables:
OMNIBUS_PROJECT: asdf
-dotnet-3:
+dotnet-3.1.302:
extends: .deb
variables:
OMNIBUS_PROJECT: dotnet
DOTNET_VERSION: '3.1.302'
-gem:
+license_management:
extends: .deb
variables:
OMNIBUS_PROJECT: license_management
-golang-1-14:
+golang-1.14.6:
extends: .deb
variables:
OMNIBUS_PROJECT: golang
@@ -52,151 +52,151 @@ java-14:
OMNIBUS_PROJECT: java
JAVA_VERSION: '14'
-mono-6:
+mono-6.8.0.123:
extends: .deb
variables:
OMNIBUS_PROJECT: mono
MONO_VERSION: '6.8.0.123'
-nodejs-10:
+nodejs-10.21.0:
extends: .deb
variables:
OMNIBUS_PROJECT: nodejs
NODE_VERSION: '10.21.0'
-nodejs-12:
+nodejs-12.18.2:
extends: .deb
variables:
OMNIBUS_PROJECT: nodejs
NODE_VERSION: '12.18.2'
-php-7:
+php-7.4.8:
extends: .deb
variables:
OMNIBUS_PROJECT: php
PHP_VERSION: '7.4.8'
-python-2-7:
+python-2.7.18:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '2.7.18'
-python-3-8:
+python-3.8.5:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '3.8.5'
-python-3-7:
+python-3.7.7:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '3.7.7'
-python-3-6:
+python-3.6.11:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '3.6.11'
-python-3-5:
+python-3.5.9:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '3.5.9'
-python-3-4:
+python-3.4.10:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '3.4.10'
-python-3-3:
+python-3.3.7:
extends: .deb
variables:
OMNIBUS_PROJECT: python
PYTHON_VERSION: '3.3.7'
-ruby-2-7-1:
+ruby-2.7.1:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.7.1'
-ruby-2-7-0:
+ruby-2.7.0:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.7.0'
-ruby-2-6-6:
+ruby-2.6.6:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.6'
-ruby-2-6-5:
+ruby-2.6.5:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.5'
-ruby-2-6-4:
+ruby-2.6.4:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.4'
-ruby-2-6-3:
+ruby-2.6.3:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.3'
-ruby-2-6-2:
+ruby-2.6.2:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.2'
-ruby-2-6-1:
+ruby-2.6.1:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.1'
-ruby-2-6-0:
+ruby-2.6.0:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.6.0'
-ruby-2-5-8:
+ruby-2.5.8:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.5.8'
-ruby-2-4-10:
+ruby-2.4.10:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.4.10'
-ruby-2-4-9:
+ruby-2.4.9:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.4.9'
-ruby-2-4-5:
+ruby-2.4.5:
extends: .deb
variables:
OMNIBUS_PROJECT: ruby
RUBY_VERSION: '2.4.5'
-rust-1-45:
+rust-1.45.0:
extends: .deb
variables:
OMNIBUS_PROJECT: rust
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 62ad140..b798258 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
# GitLab License management changelog
+## v3.25.0
+
+- Install tools from Debian package. (!188)
+
## v3.24.0
- Removes `BSD-4-Clause` from a list of normalized licenses. (!196)
diff --git a/Dockerfile b/Dockerfile
index 6cc2621..7d8ffb7 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -6,9 +6,19 @@ ENV TERM="xterm"
WORKDIR /opt/gitlab
COPY config/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc
RUN mkdir -p /opt/toolcache
+ADD https://rubygems.org/downloads/bundler-1.17.3.gem /opt/toolcache/
+ADD https://rubygems.org/downloads/bundler-2.1.4.gem /opt/toolcache/
+COPY pkg/asdf*.deb /opt/toolcache/
+COPY pkg/dotnet*.deb /opt/toolcache/
+COPY pkg/golang*.deb /opt/toolcache/
+COPY pkg/java-8*.deb /opt/toolcache/
+COPY pkg/java-11*.deb /opt/toolcache/
COPY pkg/license*.deb /opt/toolcache/
COPY pkg/mono*.deb /opt/toolcache/
+COPY pkg/node*.deb /opt/toolcache/
COPY pkg/php*.deb /opt/toolcache/
+COPY pkg/python*.deb /opt/toolcache/
+COPY pkg/ruby*.deb /opt/toolcache/
COPY pkg/rust*.deb /opt/toolcache/
COPY config/install.sh /opt/install.sh
RUN bash /opt/install.sh
diff --git a/Gemfile b/Gemfile
index 07650c5..134adcf 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
source 'https://rubygems.org'
gemspec
diff --git a/Gemfile.lock b/Gemfile.lock
index e505b27..8a88ca9 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -8,7 +8,7 @@ GIT
PATH
remote: .
specs:
- license-management (3.23.0)
+ license-management (3.25.0)
license_finder (~> 6.7)
GEM
diff --git a/bin/docker-omnibus b/bin/docker-omnibus
new file mode 100755
index 0000000..2c3e456
--- /dev/null
+++ b/bin/docker-omnibus
@@ -0,0 +1,10 @@
+#!/bin/sh
+
+set -e
+
+cd "$(dirname "$0")/.."
+
+docker run --rm -it \
+ --network=host \
+ --volume="$PWD":/build \
+ debian:stable sh -c 'cd /build && ./bin/omnibus setup && bash -l'
diff --git a/bin/omnibus b/bin/omnibus
index c46155c..5dd6c58 100755
--- a/bin/omnibus
+++ b/bin/omnibus
@@ -1,6 +1,7 @@
#!/bin/sh
set -e
+export LANG=C.UTF-8
cd "$(dirname "$0")/.."
@@ -74,4 +75,20 @@ case $1 in
for i in "$@"; do :; done
bundle exec omnibus build -l debug "$i"
;;
+
+ pull)
+ mkdir -p pkg/
+ repo_url="https://gitlab.com/gitlab-org/security-products/license-management/-/jobs/artifacts/master/raw/pkg"
+
+ grep '.*-.*:' < .gitlab/deb.yml | sed 's/://' | while IFS= read -r job
+ do
+ version="$(echo "$job" | cut -d'-' -f2)"
+ if echo "$job" | grep '^asdf.*'; then
+ name="$(echo "$job" | cut -d'-' -f1)"
+ wget -P pkg/ "${repo_url}/${name}_${version}-1_amd64.deb?job=${job}"
+ else
+ wget -P pkg/ "${repo_url}/${job}_${version}-1_amd64.deb?job=${job}"
+ fi
+ done
+ ;;
esac
diff --git a/config/files/.bashrc b/config/files/.bashrc
index b37ae94..7017be5 100644
--- a/config/files/.bashrc
+++ b/config/files/.bashrc
@@ -7,7 +7,6 @@ export HOME="/opt/gitlab"
alias nuget='mono /usr/local/bin/nuget.exe'
set -o vi
-
function inflate() {
local file=$1
local to_dir=$2
@@ -59,14 +58,13 @@ function enable_dev_mode() {
apt-get install -y --no-install-recommends vim less shellcheck
}
-inflate /opt/asdf.tar.zst /opt
-inflate /opt/gitlab/.cache.tar.zst /opt/gitlab
+if [ ! -d "/opt/asdf" ]; then
+ dpkg -i /opt/toolcache/asdf_*.deb
+fi
inflate /opt/gitlab/.config.tar.zst /opt/gitlab
inflate /opt/gitlab/.m2.tar.zst /opt/gitlab
inflate /opt/gitlab/embedded.tar.zst /opt/gitlab
inflate /usr/include.tar.zst /usr
-inflate /usr/lib/elixir.tar.zst /usr/lib
-inflate /usr/lib/erlang.tar.zst /usr/lib
inflate /usr/lib/gcc.tar.zst /usr/lib
inflate /usr/lib/git-core.tar.zst /usr/lib
inflate /usr/lib/llvm-7.tar.zst /usr/lib
diff --git a/config/files/.default-python-packages b/config/files/.default-python-packages
index ddf6574..495b9f1 100644
--- a/config/files/.default-python-packages
+++ b/config/files/.default-python-packages
@@ -1,4 +1,4 @@
conan
-pip
+pip<20.0
pipenv
virtualenv
diff --git a/config/files/.tool-versions b/config/files/.tool-versions
index 5da7afd..6fa1c96 100644
--- a/config/files/.tool-versions
+++ b/config/files/.tool-versions
@@ -1,11 +1,11 @@
-elixir system
+dotnet-core 3.1.302
+elixir 1.10.4
golang 1.14.6
-gradle 6.3
-java adoptopenjdk-8.0.252+9.1 adoptopenjdk-11.0.7+10.1
-maven 3.6.3
-nodejs 12.16.3 10.20.1
+java 8
+mono 6.8.0.123
+nodejs 12.18.2
php 7.4.8
-python 3.8.2 2.7.18
+python 3.8.5
ruby 2.6.6
rust 1.45.0
sbt 1.3.8
diff --git a/config/install.sh b/config/install.sh
index 1d0189e..3e4c581 100644
--- a/config/install.sh
+++ b/config/install.sh
@@ -8,7 +8,6 @@ apt-get clean
apt-get update -q
apt-get install -y --no-install-recommends \
apt-transport-https \
- apt-utils \
autoconf \
automake \
bison \
@@ -17,16 +16,10 @@ apt-get install -y --no-install-recommends \
bzip2 \
ca-certificates \
cmake \
- coreutils \
curl \
default-libmysqlclient-dev \
- dirmngr \
- elixir \
- gettext \
git \
- gnupg \
gnupg2 \
- gpg \
jq \
libbz2-dev \
libcurl4 \
@@ -51,70 +44,25 @@ apt-get install -y --no-install-recommends \
libssl-dev \
libtool \
libxml2-dev \
+ libxmlsec1-dev \
libxslt-dev \
libyaml-dev \
libzip-dev \
- llvm \
- locate \
make \
- openssl \
pkg-config \
- python-openssl \
re2c \
rebar \
software-properties-common \
sudo \
- tk-dev \
- unixodbc-dev \
unzip \
- wget \
- xz-utils \
zlib1g \
zlib1g-dev \
zstd
-echo -e "section_end:$(date +%s):install_packages\r\e[0K"
-echo -e "section_start:$(date +%s):install_asdf\r\e[0K==> Installing asdf…"
dpkg --install /opt/toolcache/license*.deb
rm -fr /root
ln -s /opt/gitlab /root
-
-mkdir -p "$ASDF_DATA_DIR"
-git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR"
-cd "$ASDF_DATA_DIR"
-git checkout "$(git describe --abbrev=0 --tags)"
-
-# shellcheck source=/dev/null
-. "$ASDF_DATA_DIR"/asdf.sh
-
-while IFS= read -r line; do
- tool=$(echo "$line" | cut -d' ' -f1)
- asdf plugin-add "$tool"
-done < "/opt/gitlab/.tool-versions"
-bash "$ASDF_DATA_DIR/plugins/nodejs/bin/import-release-team-keyring"
-asdf install
-rm -fr "$ASDF_DATA_DIR/installs/rust"
-rm -fr "$ASDF_DATA_DIR/installs/php"
-asdf reshim
-asdf current
-
-for version in $(asdf list python); do
- asdf shell python "$version"
- pip download -d "/opt/gitlab/.config/virtualenv/app-data" pip-licenses pip setuptools wheel
-done
-wait
-echo -e "section_end:$(date +%s):install_asdf\r\e[0K"
-
-echo -e "section_start:$(date +%s):install_dotnet\r\e[0K==> Installing dotnet/mono…"
-wget -qO- https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.asc.gpg
-wget -q -O /etc/apt/sources.list.d/microsoft-prod.list https://packages.microsoft.com/config/debian/10/prod.list
-apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF
-echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list
-
-curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe
-curl -o /tmp/dotnet-install.sh https://dotnet.microsoft.com/download/dotnet-core/scripts/v1/dotnet-install.sh
-bash /tmp/dotnet-install.sh --install-dir /opt/asdf/installs/dotnet/latest --channel LTS --version latest
-echo -e "section_end:$(date +%s):install_dotnet\r\e[0K"
+echo -e "section_end:$(date +%s):install_packages\r\e[0K"
echo -e "section_start:$(date +%s):cleanup\r\e[0K==> Beginning cleanup…"
apt purge -y libx11-6 libwayland-client0
@@ -125,16 +73,7 @@ mkdir -p /tmp
chmod 777 /tmp
chmod +t /tmp
-rm -fr "$ASDF_DATA_DIR/docs" \
- "$ASDF_DATA_DIR"/installs/golang/**/go/test \
- "$ASDF_DATA_DIR"/installs/java/**/demo \
- "$ASDF_DATA_DIR"/installs/java/**/man \
- "$ASDF_DATA_DIR"/installs/java/**/sample \
- "$ASDF_DATA_DIR"/installs/python/**/lib/**/test \
- "$ASDF_DATA_DIR"/installs/ruby/**/lib/ruby/gems/**/cache \
- "$ASDF_DATA_DIR"/installs/**/**/share \
- "$ASDF_DATA_DIR"/test \
- /opt/gitlab/.config/configstore/update-notifier-npm.json \
+rm -fr /opt/gitlab/.config/configstore/update-notifier-npm.json \
/opt/gitlab/.config/pip/selfcheck.json \
/opt/gitlab/.gem \
/opt/gitlab/.npm \
@@ -180,11 +119,8 @@ echo -e "section_end:$(date +%s):cleanup\r\e[0K"
echo -e "section_start:$(date +%s):compress_files\r\e[0K==> Starting compression…"
zstd_command="/usr/bin/zstd -19 -T0"
-cd /opt
-tar --use-compress-program "$zstd_command" -cf /opt/asdf.tar.zst asdf &
cd /opt/gitlab
-tar --use-compress-program "$zstd_command" -cf /opt/gitlab/.cache.tar.zst .cache &
tar --use-compress-program "$zstd_command" -cf /opt/gitlab/.config.tar.zst .config &
tar --use-compress-program "$zstd_command" -cf /opt/gitlab/.m2.tar.zst .m2 &
tar --use-compress-program "$zstd_command" -cf /opt/gitlab/embedded.tar.zst embedded &
@@ -194,8 +130,6 @@ tar --use-compress-program "$zstd_command" -cf /usr/include.tar.zst include &
tar --use-compress-program "$zstd_command" -cf /usr/share.tar.zst share &
cd /usr/lib
-tar --use-compress-program "$zstd_command" -cf /usr/lib/elixir.tar.zst elixir &
-tar --use-compress-program "$zstd_command" -cf /usr/lib/erlang.tar.zst erlang &
tar --use-compress-program "$zstd_command" -cf /usr/lib/gcc.tar.zst gcc &
tar --use-compress-program "$zstd_command" -cf /usr/lib/git-core.tar.zst git-core &
tar --use-compress-program "$zstd_command" -cf /usr/lib/llvm-7.tar.zst llvm-7 &
@@ -209,8 +143,6 @@ rm -fr \
/opt/gitlab/.config \
/opt/gitlab/embedded \
/usr/include \
- /usr/lib/elixir \
- /usr/lib/erlang \
/usr/lib/gcc \
/usr/lib/git-core \
/usr/lib/llvm-7 \
diff --git a/config/scripts/asdf/postinst b/config/scripts/asdf/postinst
new file mode 100755
index 0000000..ceed3e5
--- /dev/null
+++ b/config/scripts/asdf/postinst
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+tar -xf /opt/asdf/plugins.tar.gz
diff --git a/config/scripts/license_management/postinst b/config/scripts/license_management/postinst
index da63d3b..06de8f5 100755
--- a/config/scripts/license_management/postinst
+++ b/config/scripts/license_management/postinst
@@ -1,14 +1,3 @@
#!/bin/sh
-PROGNAME=$(basename "$0")
-
-error_exit()
-{
- echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2
- exit 1
-}
-
ln -s /opt/gitlab/bin/license_management /usr/local/bin/license_management
-echo "Thank you for installing license_management!"
-
-exit 0
diff --git a/config/scripts/license_management/postrm b/config/scripts/license_management/postrm
index 5b47b27..ab1a69a 100755
--- a/config/scripts/license_management/postrm
+++ b/config/scripts/license_management/postrm
@@ -1,6 +1,3 @@
#!/bin/sh
rm -f /usr/local/bin/license_management
-echo "license_management has been uninstalled!"
-
-exit 0
diff --git a/config/software/asdf.rb b/config/software/asdf.rb
index 1b5ba26..72b77ce 100644
--- a/config/software/asdf.rb
+++ b/config/software/asdf.rb
@@ -44,4 +44,6 @@ end
build do
delete "#{install_dir}/installs/elixir/**/man"
+ command "tar -czvf #{install_dir}/plugins.tar.gz #{install_dir}/plugins"
+ delete "#{install_dir}/plugins"
end
diff --git a/config/software/asdf_golang.rb b/config/software/asdf_golang.rb
index d8240b0..d7e8fd2 100644
--- a/config/software/asdf_golang.rb
+++ b/config/software/asdf_golang.rb
@@ -9,6 +9,12 @@ relative_path "go"
version "1.14.6" do
source sha256: "5c566ddc2e0bcfc25c26a5dc44a440fcc0177f7350c1f01952b34d5989a0d287"
end
+version "1.14.7" do
+ source sha256: "4a7fa60f323ee1416a4b1425aefc37ea359e9d64df19c326a58953a97ad41ea5"
+end
+version "1.15" do
+ source sha256: "2d75848ac606061efe52a8068d0e647b35ce487a15bb52272c427df485193602"
+end
build do
mkdir install_dir
diff --git a/config/software/asdf_python.rb b/config/software/asdf_python.rb
index 5a0e9c9..68cfd68 100644
--- a/config/software/asdf_python.rb
+++ b/config/software/asdf_python.rb
@@ -54,35 +54,35 @@ build do
configure(*configure_command, env: env)
make "-j #{workers}", env: env
make "-j #{workers} install", env: env
- command "ln -s #{install_dir}/embedded/bin #{install_dir}/bin"
-end
+ command "ln -s #{install_dir}/embedded/bin #{install_dir}/bin", env: env
-build do
if version.start_with?('3.')
- command "ln -s #{install_dir}/bin/python3 #{install_dir}/bin/python"
- command "ln -s #{install_dir}/bin/pip3 #{install_dir}/bin/pip"
+ command "ln -s #{install_dir}/bin/python3 #{install_dir}/bin/python", env: env
+ command "ln -s #{install_dir}/bin/pip3 #{install_dir}/bin/pip", env: env
end
- if File.exist?("#{install_dir}/bin/pip")
- command "#{install_dir}/bin/pip install --upgrade pip"
+ case version
+ when /^3\.3\./
+ command "curl https://bootstrap.pypa.io/3.3/get-pip.py -o #{project_dir}/get-pip.py", env: env
+ command "#{install_dir}/bin/python #{project_dir}/get-pip.py \"pip==10.0.1\"", env: env
+ command "#{install_dir}/bin/pip install pipenv virtualenv", env: env
+ when /^3\.4\./
+ command "#{install_dir}/bin/pip install pipenv==11.1.6 virtualenv", env: env
+ when /^2\.7\./
+ command "curl https://bootstrap.pypa.io/2.6/get-pip.py -o #{project_dir}/get-pip.py", env: env
+ command "#{install_dir}/bin/python #{project_dir}/get-pip.py \"pip<20.0\"", env: env
+ command "#{install_dir}/bin/pip install pipenv virtualenv", env: env
else
- case version
- when /^3\.2\./
- command "curl https://bootstrap.pypa.io/3.2/get-pip.py -o #{project_dir}/get-pip.py"
- when /^3\.3\./
- command "curl https://bootstrap.pypa.io/3.3/get-pip.py -o #{project_dir}/get-pip.py"
- when /^3\.4\./
- command "curl https://bootstrap.pypa.io/3.4/get-pip.py -o #{project_dir}/get-pip.py"
- when /^2\.6\./
- command "curl https://bootstrap.pypa.io/2.6/get-pip.py -o #{project_dir}/get-pip.py"
- else
- command "curl https://bootstrap.pypa.io/get-pip.py -o #{project_dir}/get-pip.py"
- end
+ command "#{install_dir}/bin/pip install pipenv virtualenv", env: env
+ end
- command "#{install_dir}/bin/python #{project_dir}/get-pip.py \"pip<20.0\""
+ if version == default_version
+ command "#{install_dir}/bin/pip install conan", env: env
+ command "#{install_dir}/bin/conan --version", env: env
end
- command "#{install_dir}/bin/pip install pipenv virtualenv"
- command "#{install_dir}/bin/pip install conan" if version == default_version
+ command "#{install_dir}/bin/pip --version", env: env
+ command "#{install_dir}/bin/pipenv --version", env: env
+ command "#{install_dir}/bin/virtualenv --version", env: env
end
build do
diff --git a/config/software/license_management.rb b/config/software/license_management.rb
index de8a854..3880799 100644
--- a/config/software/license_management.rb
+++ b/config/software/license_management.rb
@@ -38,8 +38,8 @@ end
build do
touch "#{install_dir}/.config/virtualenv/app-data/.keep"
- command "pip2 download -d #{install_dir}/.config/virtualenv/app-data pip-licenses pip setuptools wheel"
- command "pip3 download -d #{install_dir}/.config/virtualenv/app-data pip-licenses pip setuptools wheel"
+ command "pip2 download -d #{install_dir}/.config/virtualenv/app-data pip-licenses setuptools wheel"
+ command "pip3 download -d #{install_dir}/.config/virtualenv/app-data pip-licenses setuptools wheel"
end
build do
diff --git a/lib/license/finder/ext/bower.rb b/lib/license/finder/ext/bower.rb
index 9b302f6..c034568 100644
--- a/lib/license/finder/ext/bower.rb
+++ b/lib/license/finder/ext/bower.rb
@@ -1,33 +1,38 @@
# frozen_string_literal: true
module LicenseFinder
- class Bower < PackageManager
- def prepare
- shell.execute([
- :bower,
- :install,
- '--allow-root',
- '--production',
- '--verbose',
- '--loglevel',
- :debug
- ], env: default_env)
+ class Bower
+ def possible_package_paths
+ [project_path.join('bower.json')]
end
- def current_packages
- map_all(bower_output).flatten.compact
+ def prepare
+ within_project_path do
+ tool_box.install(tool: :nodejs, env: default_env)
+ shell.execute([
+ :bower,
+ :install,
+ '--allow-root',
+ '--production',
+ '--verbose',
+ '--loglevel',
+ :debug
+ ], env: default_env)
+ end
end
- def possible_package_paths
- [project_path.join('bower.json')]
+ def current_packages
+ within_project_path do
+ map_all(bower_output).flatten.compact
+ end
end
private
def bower_output
- stdout, _stderr, status = Dir.chdir(project_path) do
- shell.execute([:bower, :list, '--json', '-l', 'action', '--allow-root'], env: default_env)
- end
+ stdout, _stderr, status = shell.execute([
+ :bower, :list, '--json', '-l', 'action', '--allow-root'
+ ], env: default_env)
return {} unless status.success?
JSON.parse(stdout)
@@ -62,9 +67,5 @@ module LicenseFinder
'bower_directory' => ENV.fetch('bower_directory', vendor_path.join('bower_components')).to_s
}
end
-
- def vendor_path
- Pathname.pwd.join('.gitlab', 'cache', 'vendor')
- end
end
end
diff --git a/lib/license/finder/ext/bundler.rb b/lib/license/finder/ext/bundler.rb
index 0530f32..b8c755b 100644
--- a/lib/license/finder/ext/bundler.rb
+++ b/lib/license/finder/ext/bundler.rb
@@ -1,14 +1,25 @@
# frozen_string_literal: true
module LicenseFinder
- class Bundler < PackageManager
+ class Bundler
def prepare
create_vendor_path
with_clean_bundler do
- _stdout, _stderr, status = shell.execute([:asdf, :current, :ruby], env: default_env)
- shell.execute([:asdf, :install], env: default_env) unless status.success?
- shell.execute([:bundle, :config, '--local', :path, vendor_path.to_s], env: default_env)
+ tool_box.install(tool: :ruby, version: ruby_version, env: default_env)
+ Pathname.glob('/opt/toolcache/*.gem').each do |gem|
+ shell.execute([
+ :gem, :install, gem,
+ '--no-document',
+ '--verbose',
+ '--no-update-sources',
+ '--ignore-dependencies',
+ '--no-suggestions',
+ '--local'
+ ], env: default_env)
+ end
+ shell.execute([:asdf, :reshim], env: default_env)
+ shell.execute([:bundle, :config, '--local', :path, vendor_path], env: default_env)
shell.execute([:bundle, :install, '--verbose'], env: default_env)
end
end
@@ -53,11 +64,11 @@ module LicenseFinder
def scan_command
[
- '/opt/asdf/shims/ruby',
+ :ruby,
'-W0',
- ::License::Management.root.join('exe', 'scan_bundler').to_s,
- detected_package_path.to_s,
- lockfile_path.to_s
+ ::License::Management.root.join('exe', 'scan_bundler'),
+ detected_package_path,
+ lockfile_path
]
end
@@ -91,6 +102,10 @@ module LicenseFinder
end
end
+ def ruby_version
+ @ruby_version ||= gemfile_ruby_version || tool_box.version_of(:ruby)
+ end
+
def map_from(gem)
Dependency.new(
'Bundler',
@@ -104,5 +119,15 @@ module LicenseFinder
summary: gem[:summary]
)
end
+
+ def gemfile_ruby_version
+ with_clean_bundler do
+ stdout, _stderr, status = shell.execute([:embedded_bundle, :platform, '--ruby'])
+ return unless status.success?
+
+ match = stdout.chomp.match(/\Aruby (?<version>\d+\.\d+.\d+).*\z/)
+ match[:version] if match
+ end
+ end
end
end
diff --git a/lib/license/finder/ext/cargo.rb b/lib/license/finder/ext/cargo.rb
index 20b1b03..c957e0e 100644
--- a/lib/license/finder/ext/cargo.rb
+++ b/lib/license/finder/ext/cargo.rb
@@ -12,9 +12,8 @@ module LicenseFinder
create_vendor_path
within_project_path do
- shell.execute([:dpkg, '-i', '/opt/toolcache/rust*.deb'])
- shell.execute([:asdf, :reshim])
- shell.execute([:env, '|', :sort, '&&', :asdf, :exec, :cargo, :fetch, '-vv'], env: default_env)
+ tool_box.install(tool: :rust)
+ shell.execute([:cargo, :fetch, '-vv'], env: default_env)
end
end
@@ -44,7 +43,6 @@ module LicenseFinder
def scan_command
[
- :asdf, :exec,
:cargo, :tree, '--edges=no-dev',
"--color=never",
"--offline",
diff --git a/lib/license/finder/ext/composer.rb b/lib/license/finder/ext/composer.rb
index 1434798..993119c 100644
--- a/lib/license/finder/ext/composer.rb
+++ b/lib/license/finder/ext/composer.rb
@@ -6,6 +6,7 @@ module LicenseFinder
create_vendor_path
within_project_path do
+ tool_box.install(tool: :php)
shell.execute([
:composer,
:install,
diff --git a/lib/license/finder/ext/conan.rb b/lib/license/finder/ext/conan.rb
index ca2ee03..780b205 100644
--- a/lib/license/finder/ext/conan.rb
+++ b/lib/license/finder/ext/conan.rb
@@ -6,10 +6,16 @@ module LicenseFinder
[project_path.join('conanfile.txt')]
end
- def current_packages
- stdout, _stderr, status = Dir.chdir(project_path) do
+ def prepare
+ within_project_path do
+ tool_box.install(tool: :python)
shell.execute([:conan, :install, '--build=missing', '.'], env: default_env)
shell.execute([:conan, :inspect, '.'], env: default_env)
+ end
+ end
+
+ def current_packages
+ stdout, _stderr, status = within_project_path do
shell.execute([:conan, :info, '-j', '/dev/stdout', '.'], env: default_env)
end
return [] unless status.success?
diff --git a/lib/license/finder/ext/dotnet.rb b/lib/license/finder/ext/dotnet.rb
index 1a7eedb..eebbbbd 100644
--- a/lib/license/finder/ext/dotnet.rb
+++ b/lib/license/finder/ext/dotnet.rb
@@ -10,17 +10,19 @@ module LicenseFinder
end
def installed?(*)
- File.exist?('/opt/asdf/installs/dotnet/latest/dotnet')
+ true
end
def prepare
create_vendor_path
+ tool_box.install(tool: :"dotnet-core", version: dotnet_version)
+
shell.execute([
- '/opt/asdf/installs/dotnet/latest/dotnet',
- :restore, detected_package_path.to_s,
+ "/opt/asdf/installs/dotnet-core/#{dotnet_version}/dotnet",
+ :restore, detected_package_path,
'--locked-mode',
'--no-cache',
- '--packages', vendor_path.to_s,
+ '--packages', vendor_path,
'--verbosity', :normal
])
end
@@ -35,6 +37,10 @@ module LicenseFinder
end
end
+ def dotnet_version
+ @dotnet_version ||= tool_box.version_of(:"dotnet-core")
+ end
+
private
def map_from(name, version, data)
diff --git a/lib/license/finder/ext/go_modules.rb b/lib/license/finder/ext/go_modules.rb
index d22c59c..63b4772 100644
--- a/lib/license/finder/ext/go_modules.rb
+++ b/lib/license/finder/ext/go_modules.rb
@@ -8,7 +8,10 @@ module LicenseFinder
def prepare
return if vendored?
- shell.execute([:go, :mod, :download, '-json'])
+ within_project_path do
+ tool_box.install(tool: :golang)
+ shell.execute([:go, :mod, :download, '-json'])
+ end
end
def current_packages
diff --git a/lib/license/finder/ext/gradle.rb b/lib/license/finder/ext/gradle.rb
index 8328300..4d6b000 100644
--- a/lib/license/finder/ext/gradle.rb
+++ b/lib/license/finder/ext/gradle.rb
@@ -2,6 +2,12 @@
module LicenseFinder
class Gradle
+ def prepare
+ within_project_path do
+ tool_box.install(tool: :java, version: java_version, env: default_env)
+ end
+ end
+
def current_packages
return [] unless download_licenses
@@ -17,13 +23,21 @@ module LicenseFinder
private
+ def java_version(env: ENV)
+ @java_version ||= tool_box.version_of(:java, env: env)
+ end
+
def download_licenses
- _stdout, _stderr, status = Dir.chdir(project_path) do
+ _stdout, _stderr, status = within_project_path do
+ env = {
+ 'JAVA_HOME' => ENV.fetch("JAVA_HOME", "/opt/asdf/installs/java/#{java_version}"),
+ 'TERM' => 'noop'
+ }
shell.execute([
@command,
ENV.fetch('GRADLE_CLI_OPTS', '--exclude-task=test --no-daemon --debug'),
'downloadLicenses'
- ], env: { 'TERM' => 'noop' })
+ ], env: env)
end
status.success?
@@ -46,5 +60,12 @@ module LicenseFinder
def map_from(hash)
Dependency.from(GradlePackage.new(hash, include_groups: @include_groups), detected_package_path)
end
+
+ def default_env
+ @default_env = {
+ 'CACHE_DIR' => '/opt/gitlab',
+ 'JAVA_HOME' => ENV.fetch("JAVA_HOME", "/opt/asdf/installs/java/#{java_version}")
+ }
+ end
end
end
diff --git a/lib/license/finder/ext/maven.rb b/lib/license/finder/ext/maven.rb
index 6c95b1d..ae8c9a2 100644
--- a/lib/license/finder/ext/maven.rb
+++ b/lib/license/finder/ext/maven.rb
@@ -1,7 +1,7 @@
# frozen_string_literal: true
module LicenseFinder
- Maven.class_eval do
+ class Maven
XML_PARSE_OPTIONS = {
'ForceArray' => %w[license dependency],
'GroupTags' => {
@@ -10,10 +10,15 @@ module LicenseFinder
}
}.freeze
+ def prepare
+ within_project_path do
+ tool_box.install(tool: :java, version: java_version, env: default_env)
+ end
+ end
+
def current_packages
- Dir.chdir(project_path) do
- _stdout, _stderr, status = shell.execute(detect_licenses_command)
- return [] unless status.success?
+ within_project_path do
+ return [] unless shell.execute(detect_licenses_command, env: default_env)[-1].success?
resource_files.flat_map { |file| map_from(file.read) }.uniq
end
@@ -21,9 +26,21 @@ module LicenseFinder
private
+ def java_version(env: ENV)
+ @java_version ||= tool_box.version_of(:java, env: env)
+ end
+
+ def default_env
+ @default_env = {
+ 'CACHE_DIR' => '/opt/gitlab',
+ 'JAVA_HOME' => ENV.fetch("JAVA_HOME", "/opt/asdf/installs/java/#{java_version}")
+ }
+ end
+
def detect_licenses_command
+ mvn_wrapper = project_path.join('mvnw')
[
- package_management_command,
+ mvn_wrapper.exist? ? mvn_wrapper : :mvn,
"-e",
"org.codehaus.mojo:license-maven-plugin:aggregate-download-licenses",
"-Dlicense.excludedScopes=#{@ignored_groups.to_a.join(',')}",
@@ -37,7 +54,7 @@ module LicenseFinder
end
def map_from(xml)
- ::License::Management.logger.debug(xml)
+ log.debug(xml)
XmlSimple
.xml_in(xml, XML_PARSE_OPTIONS)['dependencies']
.map { |dependency| Dependency.from(MavenPackage.new(dependency), detected_package_path) }
diff --git a/lib/license/finder/ext/npm.rb b/lib/license/finder/ext/npm.rb
index e0d4e41..e245d72 100644
--- a/lib/license/finder/ext/npm.rb
+++ b/lib/license/finder/ext/npm.rb
@@ -2,23 +2,25 @@
module LicenseFinder
class NPM
- def current_packages
- NpmPackage.packages_from_json(npm_json, detected_package_path).map do |item|
- Dependency.from(item, detected_package_path)
- end
- end
-
- def prepare_command
- lockfile? ? 'npm ci' : 'npm install --no-save'
- end
-
def possible_package_paths
[project_path.join('package.json')]
end
def prepare
- Dir.chdir(project_path) do
- shell.execute("#{prepare_command} --production", env: default_env)
+ within_project_path do
+ tool_box.install(tool: :nodejs, env: default_env)
+
+ if lockfile?
+ shell.execute([:npm, :ci, "--production"], env: default_env)
+ else
+ shell.execute([:npm, :install, '--no-save', "--production"], env: default_env)
+ end
+ end
+ end
+
+ def current_packages
+ NpmPackage.packages_from_json(npm_json, detected_package_path).map do |item|
+ Dependency.from(item, detected_package_path)
end
end
@@ -29,16 +31,16 @@ module LicenseFinder
end
def npm_json
- stdout, _stderr, status = Dir.chdir(project_path) do
+ stdout, _stderr, status = within_project_path do
shell.execute("npm list --json --long --production")
end
status.success? ? JSON.parse(stdout) : {}
end
def default_env
- return {} unless shell.custom_certificate_installed?
-
- { 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.custom_certificate_path.to_s) }
+ {
+ 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.default_certificate_path).to_s
+ }
end
end
end
diff --git a/lib/license/finder/ext/nuget.rb b/lib/license/finder/ext/nuget.rb
index 5e689d4..f292392 100644
--- a/lib/license/finder/ext/nuget.rb
+++ b/lib/license/finder/ext/nuget.rb
@@ -3,16 +3,14 @@
module LicenseFinder
class Nuget
def prepare
- shell.execute(['apt-get', :install, '-y', '/opt/toolcache/mono*.deb'])
create_vendor_path
- Dir.chdir(project_path) do
- shell.execute([
- '/opt/asdf/installs/mono/6.8.0.123/bin/cert-sync',
- shell.default_certificate_path
- ])
+
+ within_project_path do
+ tool_box.install(tool: :mono)
+ shell.execute([:cert_sync, shell.default_certificate_path])
shell.execute([
- '/opt/asdf/installs/mono/6.8.0.123/bin/mono',
- '/opt/asdf/installs/mono/6.8.0.123/bin/nuget.exe',
+ :mono,
+ :nuget,
:restore, detected_package_path,
'-LockedMode',
'-NoCache',
@@ -48,7 +46,7 @@ module LicenseFinder
::License::Management::Nuspec.new(content).licenses
end
rescue StandardError => e
- ::License::Management.logger.error(e)
+ log.error(e)
[]
end
end
diff --git a/lib/license/finder/ext/package_manager.rb b/lib/license/finder/ext/package_manager.rb
index e343745..f90a8eb 100644
--- a/lib/license/finder/ext/package_manager.rb
+++ b/lib/license/finder/ext/package_manager.rb
@@ -6,10 +6,27 @@ module LicenseFinder
::License::Management.shell
end
+ def log
+ ::License::Management.logger
+ end
+
+ def tool_box
+ @tool_box ||= ::License::Management::ToolBox.new(shell, project_path)
+ end
+
+ def prepare
+ create_vendor_path
+ return unless prepare_command
+
+ within_project_path do
+ shell.execute(prepare_command)
+ end
+ end
+
def current_packages_with_relations
current_packages
rescue StandardError => e
- ::License::Management.logger.error(e)
+ log.error(e)
raise e unless @prepare_no_fail
[]
diff --git a/lib/license/finder/ext/pip.rb b/lib/license/finder/ext/pip.rb
index b329681..7ab8d4d 100644
--- a/lib/license/finder/ext/pip.rb
+++ b/lib/license/finder/ext/pip.rb
@@ -2,13 +2,6 @@
module LicenseFinder
class Pip
- def current_packages
- return legacy_results unless virtual_env?
-
- dependencies = python.pip_licenses(detection_path: detected_package_path)
- dependencies.any? ? dependencies : legacy_results
- end
-
def possible_package_paths
path = project_path || Pathname.pwd
@@ -18,37 +11,52 @@ module LicenseFinder
]
end
- def prepare
- return install_packages if detected_package_path == @requirements_path
-
- requirements_path = detected_package_path.dirname.join('requirements.txt')
- requirements_path.write('.') unless requirements_path.exist?
- install_packages
+ def installed?(*)
+ true
end
- private
+ def prepare
+ within_project_path do
+ return install_packages if detected_package_path == @requirements_path
- def python
- @python ||= ::License::Management::Python.new
+ requirements_path = detected_package_path.dirname.join('requirements.txt')
+ requirements_path.write('.') unless requirements_path.exist?
+ install_packages
+ end
end
- def install_packages
+ def current_packages
within_project_path do
- shell.execute(['virtualenv -p', python_executable, '--activators=bash --seeder=app-data .venv'])
- shell.sh([". .venv/bin/activate", "&&", pip_install_command], env: python.default_env)
+ return legacy_results unless File.exist?('.venv/bin/activate')
+
+ dependencies = pip_licenses.map do |dependency|
+ ::LicenseFinder::Dependency.new(
+ 'Pip',
+ dependency['Name'],
+ dependency['Version'],
+ description: dependency['Description'],
+ detection_path: detected_package_path,
+ homepage: dependency['URL'],
+ spec_licenses: [dependency['License']]
+ )
+ end
+ dependencies.any? ? dependencies : legacy_results
end
end
- def pip_install_command
- [:pip, :install, '-v', '-i', python.pip_index_url, '-r', @requirements_path]
- end
+ private
- def python_executable
- '"$(asdf where python)/bin/python"'
+ def python_version(env: ENV)
+ tool_box.version_of(:python, env: env)
end
- def virtual_env?
- within_project_path { File.exist?('.venv/bin/activate') }
+ def install_packages
+ within_project_path do
+ tool_box.install(tool: :python, version: python_version, env: default_env)
+
+ shell.execute(["/opt/asdf/installs/python/#{python_version}/bin/virtualenv", '-p', 'python', '--activators=bash --seeder=app-data .venv'])
+ shell.sh([". .venv/bin/activate", "&&", 'pip', 'install', '-v', '-r', @requirements_path], env: default_env)
+ end
end
def legacy_results
@@ -58,12 +66,40 @@ module LicenseFinder
'Pip',
name,
version,
- description: spec['description'],
+ description: spec['summary'] || spec['description'],
detection_path: detected_package_path,
homepage: spec['home_page'],
spec_licenses: PipPackage.license_names_from_spec(spec)
)
end
end
+
+ def pip_output
+ stdout, _, status = shell.execute([
+ :asdf, :exec, :python,
+ LicenseFinder::BIN_PATH.join('license_finder_pip.py'),
+ detected_package_path
+ ], env: default_env)
+ return [] unless status.success?
+
+ JSON.parse(stdout).map do |package|
+ package.values_at('name', 'version', 'dependencies', 'location')
+ end
+ end
+
+ def pip_licenses
+ shell.sh([". .venv/bin/activate", '&&', 'pip', 'install', '--no-index', "--find-links /opt/gitlab/.config/virtualenv/app-data", 'pip-licenses'], env: default_env)
+ stdout, _, status = shell.sh([". .venv/bin/activate", '&&', 'pip-licenses', '--with-description', '--with-urls', '--format=json'], env: default_env)
+ status.success? ? JSON.parse(stdout[stdout.index('[')..-1]) : []
+ end
+
+ def default_env
+ @default_env ||= {
+ 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s,
+ 'PIP_DISABLE_PIP_VERSION_CHECK' => '1',
+ 'PIP_INDEX_URL' => ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/'),
+ 'PIP_NO_PYTHON_VERSION_WARNING' => '1'
+ }
+ end
end
end
diff --git a/lib/license/finder/ext/pipenv.rb b/lib/license/finder/ext/pipenv.rb
index 687c6fc..2dfad64 100644
--- a/lib/license/finder/ext/pipenv.rb
+++ b/lib/license/finder/ext/pipenv.rb
@@ -3,22 +3,43 @@
module LicenseFinder
class Pipenv
def prepare
- return unless pipfile?
+ create_vendor_path
+ within_project_path do
+ return unless pipfile?
- shell.execute([:pipenv, '--python', python.version], env: python.default_env)
- shell.execute([:pipenv, :run, :pipenv, :sync, '--pypi-mirror', python.pip_index_url], env: python.default_env)
+ tool_box.install(tool: :python, version: python_version, env: default_env)
+ shell.execute([:asdf, :exec, :pipenv, '--python', python_version], env: default_env)
+ shell.execute([:asdf, :exec, :pipenv, :sync], env: default_env)
+ end
end
def current_packages
- return legacy_results unless pipfile?
+ within_project_path do
+ return legacy_results unless pipfile?
- python.pip_licenses(detection_path: detected_package_path)
+ dependencies = pip_licenses.map do |dependency|
+ ::LicenseFinder::Dependency.new(
+ 'Pip',
+ dependency['Name'],
+ dependency['Version'],
+ description: dependency['Description'],
+ detection_path: detected_package_path,
+ homepage: dependency['URL'],
+ spec_licenses: [dependency['License']]
+ )
+ end
+ dependencies.any? ? dependencies : legacy_results
+ end
end
private
- def python
- @python ||= ::License::Management::Python.new
+ def python_version(env: ENV)
+ @python_version ||=
+ begin
+ version = lockfile_hash.dig('_meta', 'requires', 'python_version')
+ version ? tool_box.closest_match_to(tool: :python, version: version) || version : tool_box.version_of(:python, env: env)
+ end
end
def pipfile?
@@ -38,7 +59,23 @@ module LicenseFinder
end
def lockfile_hash
- @lockfile_hash ||= JSON.parse(IO.read(detected_package_path))
+ @lockfile_hash ||= JSON.parse(detected_package_path.read)
+ end
+
+ def pip_licenses
+ _, _, status = shell.sh([". .venv/bin/activate", '&&', 'pip', 'install', '--no-index', "--find-links /opt/gitlab/.config/virtualenv/app-data", 'pip-licenses'], env: default_env)
+ shell.sh([". .venv/bin/activate", '&&', 'pip', 'install', 'pip-licenses'], env: default_env) unless status.success?
+ stdout, _stderr, status = shell.sh([". .venv/bin/activate", '&&', 'pip-licenses', '--with-description', '--with-urls', '--format=json'], env: default_env)
+ status.success? ? JSON.parse(stdout[stdout.index('[')..-1]) : []
+ end
+
+ def default_env
+ @default_env ||= {
+ 'PIPENV_PYPI_MIRROR' => ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/'),
+ 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s,
+ 'PIP_DISABLE_PIP_VERSION_CHECK' => '1',
+ 'PIP_NO_PYTHON_VERSION_WARNING' => '1'
+ }
end
end
end
diff --git a/lib/license/finder/ext/yarn.rb b/lib/license/finder/ext/yarn.rb
index 4ad97d4..e56f7d1 100644
--- a/lib/license/finder/ext/yarn.rb
+++ b/lib/license/finder/ext/yarn.rb
@@ -9,8 +9,19 @@ module LicenseFinder
[project_path.join('yarn.lock')]
end
+ def prepare
+ within_project_path do
+ tool_box.install(tool: :nodejs, version: nodejs_version, env: default_env)
+ shell.execute([
+ :yarn, :install,
+ '--ignore-engines', '--ignore-scripts',
+ '--production'
+ ], env: default_env)
+ end
+ end
+
def current_packages
- stdout, _stderr, status = Dir.chdir(project_path) do
+ stdout, _stderr, status = within_project_path do
shell.execute(list_licenses_command)
end
return [] unless status.success?
@@ -20,16 +31,6 @@ module LicenseFinder
end
end
- def prepare
- Dir.chdir(project_path) do
- shell.execute([
- :yarn, :install,
- '--ignore-engines', '--ignore-scripts',
- '--production'
- ], env: default_env)
- end
- end
-
private
def list_licenses_command
@@ -94,9 +95,13 @@ module LicenseFinder
end
def default_env
- return {} unless shell.custom_certificate_installed?
+ {
+ 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.default_certificate_path).to_s
+ }
+ end
- { 'NPM_CONFIG_CAFILE' => ENV.fetch('NPM_CONFIG_CAFILE', shell.custom_certificate_path.to_s) }
+ def nodejs_version
+ @nodejs_version ||= tool_box.version_of(:nodejs)
end
end
end
diff --git a/lib/license/management.rb b/lib/license/management.rb
index 41885d5..0b418e7 100644
--- a/lib/license/management.rb
+++ b/lib/license/management.rb
@@ -9,10 +9,10 @@ require 'license_finder'
require 'license/management/loggable'
require 'license/management/verifiable'
require 'license/management/nuspec'
-require 'license/management/python'
require 'license/management/repository'
require 'license/management/report'
require 'license/management/shell'
+require 'license/management/tool_box'
require 'license/management/version'
require 'license/finder/ext'
diff --git a/lib/license/management/python.rb b/lib/license/management/python.rb
deleted file mode 100644
index ede792e..0000000
--- a/lib/license/management/python.rb
+++ /dev/null
@@ -1,61 +0,0 @@
-# frozen_string_literal: true
-
-module License
- module Management
- class Python
- attr_reader :shell
-
- def initialize(shell: ::License::Management.shell)
- @shell = shell
- end
-
- def major_version
- version.split('.')[0]
- end
-
- def version
- ENV.fetch('ASDF_PYTHON_VERSION') do
- _stdout, stderr, status = shell.execute([:python, '--version'])
- status.success? ? stderr.split(' ')[-1] : 3
- end
- end
-
- def pip_index_url
- ENV.fetch('PIP_INDEX_URL', 'https://pypi.org/simple/')
- end
-
- def pip_licenses(venv: '.venv', detection_path:)
- _stdout, _stderr, status = shell.sh([
- ". #{venv}/bin/activate &&",
- :pip, :install,
- '--no-index',
- '--find-links /opt/gitlab/.config/virtualenv/app-data', 'pip-licenses', '&&',
- 'pip-licenses',
- '--ignore-packages prettytable',
- '--with-description',
- '--with-urls',
- '--from=meta',
- '--format=json',
- '--output-file pip-licenses.json'
- ], env: { 'PATH' => '/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin' })
- return [] unless status.success?
-
- JSON.parse(IO.read('pip-licenses.json')).map do |dependency|
- ::LicenseFinder::Dependency.new(
- 'Pip',
- dependency['Name'],
- dependency['Version'],
- description: dependency['Description'],
- detection_path: detection_path,
- homepage: dependency['URL'],
- spec_licenses: [dependency['License']]
- )
- end
- end
-
- def default_env
- { 'PIP_CERT' => ENV.fetch('PIP_CERT', shell.default_certificate_path).to_s }
- end
- end
- end
-end
diff --git a/lib/license/management/shell.rb b/lib/license/management/shell.rb
index 1a99895..b066bc1 100644
--- a/lib/license/management/shell.rb
+++ b/lib/license/management/shell.rb
@@ -4,12 +4,29 @@ module License
module Management
class Shell
SPLIT_SCRIPT = "'BEGIN {x=0;} /BEGIN CERT/{x++} { print > \"custom.\" x \".crt\" }'"
- attr_reader :default_certificate_path, :custom_certificate_path, :logger
+ COMMAND_MAP = {
+ asdf: '/opt/asdf/bin/asdf',
+ bundle: '/opt/asdf/bin/asdf exec bundle',
+ cargo: '/opt/asdf/bin/asdf exec cargo',
+ cert_sync: '/opt/asdf/installs/mono/6.8.0.123/bin/cert-sync',
+ embedded_bundle: '/opt/gitlab/embedded/bin/bundle',
+ gem: '/opt/asdf/bin/asdf exec gem',
+ go: '/opt/asdf/bin/asdf exec go',
+ keytool: '/opt/asdf/bin/asdf exec keytool',
+ mono: '/opt/asdf/installs/mono/6.8.0.123/bin/mono',
+ mvn: '/opt/asdf/bin/asdf exec mvn',
+ nuget: '/opt/asdf/installs/mono/6.8.0.123/bin/nuget.exe',
+ ruby: '/opt/asdf/bin/asdf exec ruby',
+ yarn: '/opt/asdf/bin/asdf exec yarn'
+ }.freeze
+
+ attr_reader :default_env, :default_certificate_path, :custom_certificate_path, :logger
def initialize(logger: License::Management.logger, certificate: ENV['ADDITIONAL_CA_CERT_BUNDLE'])
@logger = logger
@custom_certificate_path = Pathname.new('/usr/local/share/ca-certificates/custom.crt')
@default_certificate_path = Pathname.new('/etc/ssl/certs/ca-certificates.crt')
+ @default_env = { 'SSL_CERT_FILE' => @default_certificate_path.to_s }
trust!(certificate) if present?(certificate)
end
@@ -17,7 +34,7 @@ module License
expanded_command = expand(command)
collapsible_section(expanded_command) do
logger.debug(expanded_command)
- stdout, stderr, status = Open3.capture3(env, expanded_command)
+ stdout, stderr, status = Open3.capture3(default_env.merge(env), expanded_command)
record(stdout, stderr, status)
[stdout, stderr, status]
end
@@ -34,7 +51,10 @@ module License
private
def expand(command)
- Array(command).flatten.map(&:to_s).join(' ')
+ Array(command)
+ .flatten
+ .map { |x| COMMAND_MAP.fetch(x, x).to_s }
+ .join(' ')
end
def trust!(certificate)
@@ -44,37 +64,13 @@ module License
execute('update-ca-certificates -v')
Dir.glob('custom.*.crt').each do |path|
- full_path = File.expand_path(path)
- execute([:openssl, :x509, '-in', full_path, '-text', '-noout'])
- execute(keytool_import_command(full_path))
- execute(keytool_list_command)
+ execute([:openssl, :x509, '-in', File.expand_path(path), '-text', '-noout'])
end
end
execute([:cp, custom_certificate_path.to_s, "/usr/lib/ssl/certs/"])
execute([:c_rehash, '-v'])
end
- def keytool_import_command(file_path)
- [
- :keytool,
- '-importcert',
- '-alias', Time.now.to_i,
- '-file', file_path,
- '-trustcacerts',
- '-noprompt',
- '-storepass', 'changeit',
- '-keystore', keystore_path
- ]
- end
-
- def keytool_list_command
- [:keytool, '-list', '-v', '-storepass changeit', '-keystore', keystore_path]
- end
-
- def keystore_path
- "#{ENV['JAVA_HOME']}/jre/lib/security/cacerts"
- end
-
def present?(item)
!item.nil? && !item.empty?
end
diff --git a/lib/license/management/tool_box.rb b/lib/license/management/tool_box.rb
new file mode 100644
index 0000000..d2fe703
--- /dev/null
+++ b/lib/license/management/tool_box.rb
@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+module License
+ module Management
+ class ToolBox
+ attr_reader :project_path, :shell
+
+ def initialize(shell, project_path)
+ @shell = shell
+ @project_path = project_path
+ end
+
+ def install(tool:, version: version_of(tool), env: {})
+ Dir.chdir(project_path) do
+ deb = deb_for(tool, version)
+ if deb&.exist?
+ shell.execute([:dpkg, '-i', deb])
+ elsif tool == :nodejs
+ shell.execute(['/opt/asdf/plugins/nodejs/bin/import-release-team-keyring'])
+ end
+ shell.execute([:asdf, :install, tool.to_s, version], env: env)
+ shell.execute([:asdf, :local, tool.to_s, version], env: env)
+ shell.execute([:asdf, :reshim], env: env)
+ end
+
+ install_certificates_into_java_keystore(env) if tool == :java
+ end
+
+ def version_of(tool, env: ENV)
+ Dir.chdir(project_path) do
+ case tool
+ when :java
+ java_version(env: env)
+ when :python
+ python_version(env: env)
+ else
+ asdf_detected_version_of(tool)
+ end
+ end
+ end
+
+ def default_version_for(tool)
+ default_versions[tool.to_sym]
+ end
+
+ def uninstall(tool:, version:)
+ Dir.chdir(project_path) do
+ return unless deb_for(tool, version)&.exist?
+
+ shell.execute([:dpkg, '-r', "#{tool}-#{version}"])
+ shell.execute([:asdf, :reshim])
+ end
+ end
+
+ def closest_match_to(tool:, version:)
+ deb = deb_for(tool, version)
+ deb.basename.to_s.gsub("#{tool}-", "").split('_', 2)[0] if deb&.exist?
+ end
+
+ private
+
+ def deb_for(tool, version)
+ Pathname.glob("/opt/toolcache/#{tool}-#{version}*.deb")[0]
+ end
+
+ def version_error_for(tool)
+ /\Aversion\s(?<version>.+)\sis not installed for #{tool}\z/
+ end
+
+ def default_versions
+ @default_versions ||=
+ IO.read('/opt/gitlab/.tool-versions').each_line.each_with_object({}) do |line, memo|
+ name, version = line.chomp.split(' ', 2)
+ memo[name.to_sym] = version
+ end
+ end
+
+ def asdf_detected_version_of(tool)
+ stdout, stderr, status = shell.execute([:asdf, :current, tool.to_s])
+
+ if status.success?
+ version, _ = stdout.split(' ', 2)
+ version
+ else
+ match = stderr.chomp.match(version_error_for(tool))
+ match ? match[:version] : default_version_for(tool)
+ end
+ end
+
+ def python_version(env:)
+ case env['LM_PYTHON_VERSION']
+ when '3'
+ default_version_for(:python)
+ when '2'
+ closest_match_to(tool: :python, version: "2.")
+ else
+ major, minor, _rest = (env['ASDF_PYTHON_VERSION'] || asdf_detected_version_of(:python)).split('.')
+ closest_match_to(tool: :python, version: "#{major}.#{minor}")
+ end
+ end
+
+ def java_version(env:)
+ lm_version = env['LM_JAVA_VERSION']
+ return lm_version if lm_version && %w[8 11].include?(lm_version)
+
+ asdf_version = env['ASDF_JAVA_VERSION']
+ return asdf_version.gsub('adopt-openjdk', 'adoptopenjdk') if asdf_version
+
+ asdf_detected_version_of(:java)
+ end
+
+ def install_certificates_into_java_keystore(env)
+ Dir.chdir shell.custom_certificate_path.dirname do
+ Dir.glob('custom.*.crt').each do |path|
+ keystore_path = "#{env['JAVA_HOME']}/jre/lib/security/cacerts"
+ shell.execute([:keytool, '-importcert', '-alias', Time.now.to_i, '-file', File.expand_path(path), '-trustcacerts', '-noprompt', '-storepass', 'changeit', '-keystore', keystore_path], env: env)
+ shell.execute([:keytool, '-list', '-v', '-storepass changeit', '-keystore', keystore_path], env: env)
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb
index fb91980..a9eefb3 100644
--- a/lib/license/management/version.rb
+++ b/lib/license/management/version.rb
@@ -2,6 +2,6 @@
module License
module Management
- VERSION = '3.23.0'
+ VERSION = '3.25.0'
end
end
diff --git a/omnibus.rb b/omnibus.rb
index ecd6816..7a30f95 100644
--- a/omnibus.rb
+++ b/omnibus.rb
@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
base_dir './tmp'
cache_dir './tmp/omnibus/cache'
git_cache_dir './tmp/omnibus/cache/git_cache'
diff --git a/run.sh b/run.sh
index b4d6bdd..c7eb446 100755
--- a/run.sh
+++ b/run.sh
@@ -6,8 +6,6 @@ set -e
[[ -z ${SETUP_CMD:-} ]] && set -uo pipefail
-export ASDF_JAVA_VERSION="${ASDF_JAVA_VERSION:-}"
-export ASDF_PYTHON_VERSION="${ASDF_PYTHON_VERSION:-}"
export CI_API_V4_URL="${CI_API_V4_URL:-https://gitlab.com/api/v4}"
export DEBIAN_FRONTEND=noninteractive
export DOTNET_CLI_TELEMETRY_OPTOUT=1
@@ -17,8 +15,8 @@ export HISTFILESIZE=0
export HISTSIZE=0
export LANG=C.UTF-8
export LICENSE_FINDER_CLI_OPTS=${LICENSE_FINDER_CLI_OPTS:=--no-debug}
-export LM_JAVA_VERSION=${LM_JAVA_VERSION:-}
-export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-}
+export LM_JAVA_VERSION=${LM_JAVA_VERSION:-"8"}
+export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-"3"}
export LM_REPORT_FILE=${LM_REPORT_FILE:-'gl-license-scanning-report.json'}
export MAVEN_CLI_OPTS="${MAVEN_CLI_OPTS:--DskipTests}"
export NO_UPDATE_NOTIFIER=true
@@ -40,43 +38,8 @@ function scan_project() {
license_management report $@
}
-function prepare_java() {
- if [ -n "$LM_JAVA_VERSION" ]; then
- switch_to java "adoptopenjdk-${LM_JAVA_VERSION}"
- elif [ -n "$ASDF_JAVA_VERSION" ]; then
- switch_to_exact java "${ASDF_JAVA_VERSION//adopt-openjdk/adoptopenjdk}"
- else
- switch_to java "adoptopenjdk-8"
- fi
-}
-
-function prepare_python() {
- if [ -n "$LM_PYTHON_VERSION" ]; then
- switch_to python "$(major_version_from "$LM_PYTHON_VERSION")"
- elif [ -n "$ASDF_PYTHON_VERSION" ]; then
- switch_to_exact python "$ASDF_PYTHON_VERSION"
- else
- switch_to python "3"
- fi
-}
-
-function prepare_tools() {
- dpkg -i /opt/toolcache/php*.deb || true
- dpkg -i /opt/toolcache/rust*.deb || true
-
- if ! asdf current 2> >(grep -q 'is not installed'); then
- echo "Installing missing tools…"
- asdf install
- fi
-}
-
function prepare_project() {
- prepare_java
- prepare_python
-
- if [[ -z ${SETUP_CMD:-} ]]; then
- prepare_tools || true
- else
+ if [[ -n ${SETUP_CMD:-} ]]; then
echo "Running '${SETUP_CMD}' to install project dependencies…"
# shellcheck disable=SC2068
${SETUP_CMD[@]}
diff --git a/spec/fixtures/python/pipenv/specific-python-version/.tool-versions b/spec/fixtures/python/pipenv/specific-python-version/.tool-versions
deleted file mode 100644
index 5f51665..0000000
--- a/spec/fixtures/python/pipenv/specific-python-version/.tool-versions
+++ /dev/null
@@ -1 +0,0 @@
-python 3.4.10
diff --git a/spec/integration/dotnet/nuget_spec.rb b/spec/integration/dotnet/nuget_spec.rb
index c72296e..f640c32 100644
--- a/spec/integration/dotnet/nuget_spec.rb
+++ b/spec/integration/dotnet/nuget_spec.rb
@@ -8,6 +8,7 @@ RSpec.describe "nuget" do
let(:env) { {} }
before do
+ system("rm -fr /opt/asdf/installs/mono")
system("rm -fr /opt/gitlab/.nuget/packages")
end
diff --git a/spec/integration/java/gradle_spec.rb b/spec/integration/java/gradle_spec.rb
index 25d7e26..e6bda3c 100644
--- a/spec/integration/java/gradle_spec.rb
+++ b/spec/integration/java/gradle_spec.rb
@@ -122,10 +122,10 @@ plugins {
runner.add_file('.tool-versions', "gradle 1.9")
end
- it 'returns an empty report because the plugin we use does not work in this version of the gradle API' do
+ specify do
expect(subject).to match_schema
- expect(subject[:dependencies]).to be_empty
- expect(subject[:licenses]).to be_empty
+ expect(subject.dependency_names).to match_array(['postgresql'])
+ expect(subject.licenses_for('postgresql')).to match_array(['BSD-2-Clause'])
end
end
diff --git a/spec/integration/java/maven_spec.rb b/spec/integration/java/maven_spec.rb
index 65d8bf5..217035d 100644
--- a/spec/integration/java/maven_spec.rb
+++ b/spec/integration/java/maven_spec.rb
@@ -114,48 +114,6 @@ RSpec.describe "maven" do
specify { expect(report.licenses_for('jackson-core')).to match_array(['Apache-2.0']) }
end
- context "when specifying the version of java using environment variables" do
- let(:output_file) { Pathname.new(runner.project_path.join('output.txt')) }
-
- before do
- runner.add_file('custom.sh') do
- <<~SCRIPT
- #!/bin/bash -l
-
- java -version &> '#{output_file}'
- SCRIPT
- end
- end
-
- it 'prioritizes `LM_JAVA_VERSION` over the `ASDF_JAVA_VERSION`' do
- runner.scan(env: {
- 'ASDF_JAVA_VERSION' => 'adopt-openjdk-11.0.7+10',
- 'LM_JAVA_VERSION' => '8',
- 'SETUP_CMD' => 'bash custom.sh'
- })
-
- expect(output_file).to exist
- expect(output_file.read).to include('openjdk version "1.8.0_252"')
- end
-
- it 'reads the ASDF_JAVA_VERSION' do
- runner.scan(env: {
- 'ASDF_JAVA_VERSION' => 'adopt-openjdk-11.0.7+10.1',
- 'SETUP_CMD' => 'bash custom.sh'
- })
-
- expect(output_file).to exist
- expect(output_file.read).to include('openjdk version "11.0.7"')
- end
-
- it 'defaults to java 8' do
- runner.scan(env: { 'SETUP_CMD' => 'bash custom.sh' })
-
- expect(output_file).to exist
- expect(output_file.read).to include('openjdk version "1.8.0_252"')
- end
- end
-
[
{ java: '8', maven: ['3.6.3', '3.5.4', '3.3.9', '3.2.5'] },
{ java: '11', maven: ['3.6.3', '3.5.4', '3.3.9', '3.2.5'] }
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index 9911dc5..8b7d43b 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -3,6 +3,10 @@
require 'spec_helper'
RSpec.describe "pip" do
+ before do
+ system('rm -fr /opt/asdf/installs/python')
+ end
+
context "when a project depends on the latest version of pip" do
let(:requirements) { "sentry-sdk>=0.7.7" }
@@ -65,18 +69,21 @@ RSpec.describe "pip" do
[{ version: '2', commit: '04dce91b' }, { version: '3', commit: '48e250a1' }].each do |python|
['1.0', '1.1', '2.0', '2.1'].each do |report_version|
context "when generating a `#{report_version}` report using Python `#{python[:version]}`" do
+ subject { runner.scan(env: environment) }
+
let(:url) { "https://gitlab.com/gitlab-org/security-products/tests/#{language}-#{package_manager}.git" }
let(:language) { 'python' }
let(:package_manager) { 'pip' }
let(:environment) { { 'LM_REPORT_VERSION' => report_version, 'LM_PYTHON_VERSION' => python[:version] } }
let(:expected_content) { fixture_file_content("expected/#{language}/#{python[:version]}/#{package_manager}/v#{report_version}.json").chomp }
- it 'matches the expected report' do
+ before do
runner.clone(url, branch: python[:commit])
- report = runner.scan(env: environment)
+ end
- expect(JSON.pretty_generate(report.to_h)).to eq(expected_content)
- expect(report).to match_schema(version: report_version)
+ it 'matches the expected report' do
+ expect(JSON.pretty_generate(subject.to_h)).to eq(expected_content)
+ expect(subject).to match_schema(version: report_version)
end
end
end
@@ -134,8 +141,8 @@ RSpec.describe "pip" do
report = runner.scan(env: { 'SETUP_CMD' => 'bash custom.sh' })
expect(report).to match_schema(version: '2.0')
- expect(report.licenses_for('six')).to match_array(["MIT"])
expect(report.dependency_names).to contain_exactly('six')
+ expect(report.licenses_for('six')).to match_array(["MIT"])
end
end
@@ -187,46 +194,4 @@ RSpec.describe "pip" do
expect(report.licenses_for('requests')).to match_array(['Apache-2.0'])
end
end
-
- context "when specifying the version of Python using environment variables" do
- let(:output_file) { Pathname.new(runner.project_path.join('output.txt')) }
-
- before do
- runner.add_file('custom.sh') do
- <<~SCRIPT
- #!/bin/bash -l
-
- python --version &> '#{output_file}'
- SCRIPT
- end
- end
-
- it 'prioritizes `LM_PYTHON_VERSION` over the `ASDF_PYTHON_VERSION`' do
- runner.scan(env: {
- 'ASDF_PYTHON_VERSION' => '3.8.2',
- 'LM_PYTHON_VERSION' => '2.7.18',
- 'SETUP_CMD' => 'bash custom.sh'
- })
-
- expect(output_file).to exist
- expect(output_file.read).to include('2.7.18')
- end
-
- it 'reads the ASDF_PYTHON_VERSION' do
- runner.scan(env: {
- 'ASDF_PYTHON_VERSION' => '3.8.2',
- 'SETUP_CMD' => 'bash custom.sh'
- })
-
- expect(output_file).to exist
- expect(output_file.read).to include('3.8.2')
- end
-
- it 'defaults to Python 3' do
- runner.scan(env: { 'SETUP_CMD' => 'bash custom.sh' })
-
- expect(output_file).to exist
- expect(output_file.read).to include('3.8.2')
- end
- end
end
diff --git a/spec/integration/python/pipenv_spec.rb b/spec/integration/python/pipenv_spec.rb
index cd20522..fad3edf 100644
--- a/spec/integration/python/pipenv_spec.rb
+++ b/spec/integration/python/pipenv_spec.rb
@@ -7,6 +7,10 @@ RSpec.describe "pipenv" do
let(:env) { {} }
+ before do
+ system('rm -fr /opt/asdf/installs/python')
+ end
+
include_examples "each report version", "python", "pipenv", "pip-file-lock"
context "when a project depends on a version 6 Pipfile.lock" do
diff --git a/spec/integration/ruby/bundler_spec.rb b/spec/integration/ruby/bundler_spec.rb
index 6cb5543..1c0e472 100644
--- a/spec/integration/ruby/bundler_spec.rb
+++ b/spec/integration/ruby/bundler_spec.rb
@@ -7,6 +7,10 @@ RSpec.describe "bundler" do
let(:env) { {} }
+ before do
+ system('rm -fr /opt/asdf/installs/ruby')
+ end
+
include_examples "each report version", "ruby", "bundler"
context "when the project depends on an older version of ruby specified in a `.ruby-version` file" do
@@ -89,6 +93,19 @@ RSpec.describe "bundler" do
end
end
+ context "when scanning the `gitlab-runner` project" do
+ before do
+ runner.clone('https://gitlab.com/gitlab-org/gitlab-runner.git')
+ end
+
+ specify do
+ expect(subject).to match_schema
+ expect(subject[:licenses]).not_to be_empty
+ expect(subject[:dependencies]).not_to be_empty
+ expect(subject.dependency_names).to include('gitlab.com/gitlab-org/gitlab-terminal')
+ end
+ end
+
context "when fetching dependencies from a custom registry" do
before do
add_host('rubygems.test', '127.0.0.1')
diff --git a/spec/support/shared.rb b/spec/support/shared.rb
index 298d85d..28eb8c3 100644
--- a/spec/support/shared.rb
+++ b/spec/support/shared.rb
@@ -3,6 +3,8 @@
RSpec.shared_examples "each report version" do |language, package_manager, branch = 'master'|
['1.0', '1.1', '2.0', '2.1'].each do |version|
context "when generating a `#{version}` report for #{package_manager}" do
+ subject { runner.scan(env: { 'LM_REPORT_VERSION' => version }) }
+
let(:url) { "https://gitlab.com/gitlab-org/security-products/tests/#{language}-#{package_manager}.git" }
let(:expected_content) { JSON.parse(fixture_file_content("expected/#{language}/#{package_manager}/v#{version}.json")) }
@@ -11,10 +13,8 @@ RSpec.shared_examples "each report version" do |language, package_manager, branc
end
it 'matches the expected report' do
- actual = runner.scan(env: { 'LM_REPORT_VERSION' => version })
-
- expect(JSON.pretty_generate(actual.to_h)).to eq(JSON.pretty_generate(expected_content))
- expect(actual).to match_schema(version: version)
+ expect(JSON.pretty_generate(subject.to_h)).to eq(JSON.pretty_generate(expected_content))
+ expect(subject).to match_schema(version: version)
end
end
end
diff --git a/spec/unit/license_finder/bundler_spec.rb b/spec/unit/license_finder/bundler_spec.rb
index 5458892..84edde2 100644
--- a/spec/unit/license_finder/bundler_spec.rb
+++ b/spec/unit/license_finder/bundler_spec.rb
@@ -1,26 +1,28 @@
+# frozen_string_literal: true
+
require 'spec_helper'
RSpec.describe LicenseFinder::Bundler do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
describe "#current_packages" do
subject do
project.chdir do
- bundler.prepare
- bundler.current_packages
+ package_manager.prepare
+ package_manager.current_packages
end
end
- let(:bundler) { described_class.new(options) }
- let(:options) { { ignored_groups: [], project_path: project.project_path } }
- let(:project) { ProjectHelper.new }
-
- before do
- project.mount(dir: project_fixture)
- end
-
- after do
- project.cleanup
- end
-
context "when scanning a v2.1 bundler project" do
let(:project_fixture) { fixture_file('ruby/bundler-v2.1') }
@@ -45,4 +47,32 @@ RSpec.describe LicenseFinder::Bundler do
specify { expect(subject.map(&:name)).to include("saml-kit") }
end
end
+
+ describe "#ruby_version" do
+ subject { package_manager.send(:ruby_version) }
+
+ context "when the version of ruby is specified in the Gemfile" do
+ let(:project_fixture) { fixture_file('ruby/bundler/ruby-2.4.9') }
+
+ specify { expect(subject).to eql('2.4.9') }
+ end
+
+ context "when the version of ruby is specified in a .ruby-version file" do
+ let(:project_fixture) { fixture_file('ruby/bundler-ruby-2.4.9-no-lockfile') }
+
+ specify { expect(subject).to eql('2.4.9') }
+ end
+
+ context "when the version of ruby is specified in a .tool-versions file" do
+ let(:project_fixture) { fixture_file('ruby/bundler/ruby-2.6.0-tool-versions') }
+
+ specify { expect(subject).to eql('2.6.0') }
+ end
+
+ context "when a ruby is not specified it uses the default version" do
+ let(:project_fixture) { fixture_file('ruby/bundler-v2.1') }
+
+ specify { expect(subject).to eql('2.6.6') }
+ end
+ end
end
diff --git a/spec/unit/license_finder/dotnet_spec.rb b/spec/unit/license_finder/dotnet_spec.rb
new file mode 100644
index 0000000..144d29e
--- /dev/null
+++ b/spec/unit/license_finder/dotnet_spec.rb
@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe LicenseFinder::Dotnet do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
+ describe "#dotnet_version" do
+ subject { package_manager.dotnet_version }
+
+ context "when the version of dotnet is specified in a .tool-versions file" do
+ let(:project_fixture) { fixture_file('dotnet/nuget-csproj') }
+
+ before do
+ project.add_file('.tool-versions', 'dotnet-core 3.1.301')
+ end
+
+ specify { expect(subject).to eql('3.1.301') }
+ end
+
+ context "when a nodejs version is not specified" do
+ let(:project_fixture) { fixture_file('dotnet/nuget-csproj') }
+
+ specify { expect(subject).to eql('3.1.302') }
+ end
+ end
+end
diff --git a/spec/unit/license_finder/gradle_spec.rb b/spec/unit/license_finder/gradle_spec.rb
new file mode 100644
index 0000000..18c58d7
--- /dev/null
+++ b/spec/unit/license_finder/gradle_spec.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe LicenseFinder::Gradle do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+ let(:project_fixture) { fixture_file('java/gradle/java-8') }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
+ describe "#java_version" do
+ subject { package_manager.send(:java_version, env: env) }
+
+ let(:env) { {} }
+
+ context "when the version is specified in a .tool-versions file" do
+ let(:project_fixture) { fixture_file('java/maven/tool-versions') }
+
+ specify { expect(subject).to eql('tool-versions') }
+ end
+
+ context "when the version is specified in a .java-version file" do
+ let(:project_fixture) { fixture_file('java/maven/java-version') }
+
+ specify { expect(subject).to eql('java-version') }
+ end
+
+ context "when the version is specified via a ASDF_JAVA_VERSION environment variable" do
+ let(:env) { { "ASDF_JAVA_VERSION" => 'adopt-openjdk-11' } }
+
+ specify { expect(subject).to eql('adoptopenjdk-11') }
+ end
+
+ %w[8 11].each do |java_version|
+ context "when the version is specified via a LM_JAVA_VERSION (#{java_version}) environment variable" do
+ let(:env) { { "LM_JAVA_VERSION" => java_version } }
+
+ specify { expect(subject).to eql(java_version) }
+ end
+ end
+
+ context "when LM_JAVA_VERSION and ASDF_JAVA_VERSION is provided" do
+ let(:env) do
+ {
+ 'ASDF_JAVA_VERSION' => 'adopt-openjdk-11',
+ 'LM_JAVA_VERSION' => '8'
+ }
+ end
+
+ specify { expect(subject).to eql('8') }
+ end
+
+ context 'when a custom Java version is not specified' do
+ specify { expect(subject).to eql('8') }
+ end
+ end
+end
diff --git a/spec/unit/license_finder/maven_spec.rb b/spec/unit/license_finder/maven_spec.rb
new file mode 100644
index 0000000..737b707
--- /dev/null
+++ b/spec/unit/license_finder/maven_spec.rb
@@ -0,0 +1,65 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe LicenseFinder::Maven do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+ let(:project_fixture) { fixture_file('java/maven/simple') }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
+ describe "#java_version" do
+ subject { package_manager.send(:java_version, env: env) }
+
+ let(:env) { {} }
+
+ context "when the version is specified in a .tool-versions file" do
+ let(:project_fixture) { fixture_file('java/maven/tool-versions') }
+
+ specify { expect(subject).to eql('tool-versions') }
+ end
+
+ context "when the version is specified in a .java-version file" do
+ let(:project_fixture) { fixture_file('java/maven/java-version') }
+
+ specify { expect(subject).to eql('java-version') }
+ end
+
+ context "when the version is specified via a ASDF_JAVA_VERSION environment variable" do
+ let(:env) { { "ASDF_JAVA_VERSION" => 'adopt-openjdk-11' } }
+
+ specify { expect(subject).to eql('adoptopenjdk-11') }
+ end
+
+ %w[8 11].each do |java_version|
+ context "when the version is specified via a LM_JAVA_VERSION (#{java_version}) environment variable" do
+ let(:env) { { "LM_JAVA_VERSION" => java_version } }
+
+ specify { expect(subject).to eql(java_version) }
+ end
+ end
+
+ context "when LM_JAVA_VERSION and ASDF_JAVA_VERSION is provided" do
+ let(:env) do
+ {
+ 'ASDF_JAVA_VERSION' => 'adopt-openjdk-11',
+ 'LM_JAVA_VERSION' => '8'
+ }
+ end
+
+ specify { expect(subject).to eql('8') }
+ end
+
+ context 'when a custom Java version is not specified' do
+ specify { expect(subject).to eql('8') }
+ end
+ end
+end
diff --git a/spec/unit/license_finder/pip_spec.rb b/spec/unit/license_finder/pip_spec.rb
new file mode 100644
index 0000000..8a5efbc
--- /dev/null
+++ b/spec/unit/license_finder/pip_spec.rb
@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe LicenseFinder::Pip do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+ let(:project_fixture) { fixture_file('python/pip/requirements') }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
+ describe "#python_version" do
+ subject { package_manager.send(:python_version, env: env) }
+
+ let(:env) { {} }
+
+ context "when the version is specified in a .tool-versions file" do
+ before do
+ project.add_file(".tool-versions", "python 3.8.1")
+ end
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ context "when the version is specified in a .python-version file" do
+ before do
+ project.add_file(".python-version", "3.8.0")
+ end
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ context "when the version is specified via a ASDF_PYTHON_VERSION environment variable" do
+ let(:env) { { "ASDF_PYTHON_VERSION" => '3.8.1' } }
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ [['2', '2.7.18'], ['3', '3.8.5']].each do |(major, version)|
+ context "when the version is specified via a LM_PYTHON_VERSION (#{major}) environment variable" do
+ let(:env) { { "LM_PYTHON_VERSION" => major } }
+
+ specify { expect(subject).to eql(version) }
+ end
+ end
+
+ context "when LM_PYTHON_VERSION and ASDF_PYTHON_VERSION is provided" do
+ let(:env) do
+ {
+ 'ASDF_PYTHON_VERSION' => '2.7.19',
+ 'LM_PYTHON_VERSION' => '3'
+ }
+ end
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ context 'when a custom Python version is not specified' do
+ specify { expect(subject).to eql('3.8.5') }
+ end
+ end
+end
diff --git a/spec/unit/license_finder/pipenv_spec.rb b/spec/unit/license_finder/pipenv_spec.rb
new file mode 100644
index 0000000..7971df5
--- /dev/null
+++ b/spec/unit/license_finder/pipenv_spec.rb
@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe LicenseFinder::Pipenv do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+ let(:project_fixture) { fixture_file('python/pipenv/simple') }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
+ describe "#python_version" do
+ subject { package_manager.send(:python_version, env: env) }
+
+ let(:env) { {} }
+
+ context "when the version is specified in a .tool-versions file" do
+ before do
+ project.add_file(".tool-versions", "python 3.8.1")
+ end
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ context "when the version is specified in a .python-version file" do
+ before do
+ project.add_file(".python-version", "3.8.0")
+ end
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ context "when the version is specified via a ASDF_PYTHON_VERSION environment variable" do
+ let(:env) { { "ASDF_PYTHON_VERSION" => '3.8.1' } }
+
+ specify { expect(subject).to eql('3.8.5') }
+ end
+
+ [['2', '2.7.18'], ['3', '3.8.5']].each do |(major, version)|
+ context "when the version is specified via a LM_PYTHON_VERSION (#{major}) environment variable" do
+ let(:env) { { "LM_PYTHON_VERSION" => major } }
+
+ specify { expect(subject).to eql(version) }
+ end
+ end
+
+ context "when LM_PYTHON_VERSION and ASDF_PYTHON_VERSION is provided" do
+ let(:env) do
+ {
+ 'ASDF_PYTHON_VERSION' => '2.7.19',
+ 'LM_PYTHON_VERSION' => '3'
+ }
+ end
+
+ specify { expect(subject).to start_with('3.8') }
+ end
+
+ context 'when the version is specified in the Pipfile.lock' do
+ let(:project_fixture) { fixture_file('python/pipenv/specific-python-version') }
+
+ specify { expect(subject).to start_with('3.4') }
+ end
+
+ context 'when a custom Python version is not specified' do
+ specify { expect(subject).to eql('3.8.5') }
+ end
+ end
+end
diff --git a/spec/unit/license_finder/yarn_spec.rb b/spec/unit/license_finder/yarn_spec.rb
new file mode 100644
index 0000000..7062ea3
--- /dev/null
+++ b/spec/unit/license_finder/yarn_spec.rb
@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe LicenseFinder::Yarn do
+ let(:package_manager) { described_class.new(options) }
+ let(:options) { { ignored_groups: [], project_path: project.project_path } }
+ let(:project) { ProjectHelper.new }
+
+ before do
+ project.mount(dir: project_fixture)
+ end
+
+ after do
+ project.cleanup
+ end
+
+ describe "#nodejs_version" do
+ subject { package_manager.send(:nodejs_version) }
+
+ context "when the version of nodejs is specified in a .tool-versions file" do
+ let(:project_fixture) { fixture_file('js/10.21.0-tool-versions') }
+
+ specify { expect(subject).to eql('10.21.0') }
+ end
+
+ context "when a nodejs version is not specified" do
+ let(:project_fixture) { fixture_file('js/yarn/single-declared-dependency') }
+
+ specify { expect(subject).to eql('12.18.2') }
+ end
+ end
+end