From 36cdb0040abda394264455a1fdf3d6782af95ceb Mon Sep 17 00:00:00 2001 From: mo khan Date: Mon, 6 Jul 2020 11:03:35 -0600 Subject: Isolate license_management ruby from project * Target ruby version 2.7.1 * Add spec to fetch gems from a custom source * Add proxy to rubygems.org config * Specify default env vars to support offline environment * Cleanup custom certificates after spec * Inline docker-test script * Do not install license_finder with each installed ruby * Increase gem log verbosity and include backtrace * Extract test fixtures for the different ruby scenarios * Find *.gemspec files in gems dir * Use RUBYLIB to hijack src path * Run scan from project path dir --- .dockerignore | 5 +- .gitignore | 1 + .gitlab-ci.yml | 1 - .gitlab/build.yml | 5 +- .gitlab/test.yml | 9 +- .rubocop.yml | 9 +- CHANGELOG.md | 4 + Dockerfile | 45 +++------- Gemfile | 1 + Gemfile.lock | 99 +++++++++++++++++++++- README.md | 5 +- bin/docker-build | 4 +- bin/docker-shell | 4 +- bin/lint | 4 +- bin/omnibus | 46 ++++++++++ bin/setup | 11 ++- bin/test | 9 +- config/.asdfrc | 1 - config/.bashrc | 64 -------------- config/.config/NuGet/NuGet.Config | 13 --- config/.config/virtualenv/virtualenv.ini | 4 - config/.curlrc | 2 - config/.default-gems | 3 - config/.default-npm-packages | 4 - config/.default-python-packages | 4 - config/.env.12-10-stable | 2 - config/.env.12-2-stable | 2 - config/.env.12-3-stable | 2 - config/.env.12-4-stable | 2 - config/.env.12-5-stable | 2 - config/.env.12-6-stable | 2 - config/.env.12-7-stable | 2 - config/.env.12-8-stable | 2 - config/.env.12-9-stable | 2 - config/.env.13-0-stable | 2 - config/.env.legacy | 3 - config/.gemrc | 4 - config/.gradle/init.gradle | 13 --- config/.m2/settings.xml | 32 ------- config/.npmrc | 13 --- config/.profile | 3 - config/.tool-versions | 11 --- config/files/.asdfrc | 1 + config/files/.bashrc | 69 +++++++++++++++ config/files/.config/NuGet/NuGet.Config | 13 +++ config/files/.config/virtualenv/virtualenv.ini | 4 + config/files/.curlrc | 2 + config/files/.default-gems | 2 + config/files/.default-npm-packages | 4 + config/files/.default-python-packages | 4 + config/files/.env.12-10-stable | 2 + config/files/.env.12-2-stable | 2 + config/files/.env.12-3-stable | 2 + config/files/.env.12-4-stable | 2 + config/files/.env.12-5-stable | 2 + config/files/.env.12-6-stable | 2 + config/files/.env.12-7-stable | 2 + config/files/.env.12-8-stable | 2 + config/files/.env.12-9-stable | 2 + config/files/.env.13-0-stable | 2 + config/files/.env.legacy | 3 + config/files/.gemrc | 4 + config/files/.gradle/init.gradle | 13 +++ config/files/.m2/settings.xml | 32 +++++++ config/files/.npmrc | 13 +++ config/files/.profile | 3 + config/files/.tool-versions | 11 +++ config/install.sh | 24 +++--- config/projects/license_management.rb | 17 ++++ config/scripts/license_management/postinst | 14 +++ config/scripts/license_management/postrm | 6 ++ config/scripts/license_management/preinst | 3 + config/scripts/license_management/prerm | 11 +++ config/software/license_management.rb | 34 ++++++++ config/software/ruby.rb | 73 ++++++++++++++++ exe/scan_bundler | 24 ++++++ lib/license/finder/ext/bundler.rb | 82 +++++++++++------- lib/license/finder/ext/package_manager.rb | 4 + lib/license/finder/ext/shared_helpers.rb | 4 - lib/license/management/python.rb | 2 +- lib/license/management/version.rb | 2 +- license-management.gemspec | 1 + omnibus.rb | 9 ++ pkg/.keep | 0 run.sh | 3 +- spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile | 5 ++ spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile.lock | 47 ++++++++++ spec/spec_helper.rb | 2 + spec/support/integration_test_helper.rb | 94 +------------------- spec/support/project_helper.rb | 56 ++++++++++++ spec/support/report.rb | 37 ++++++++ spec/unit/license_finder/bundler_spec.rb | 48 +++++++++++ 92 files changed, 860 insertions(+), 396 deletions(-) create mode 100755 bin/omnibus delete mode 100644 config/.asdfrc delete mode 100644 config/.bashrc delete mode 100644 config/.config/NuGet/NuGet.Config delete mode 100644 config/.config/virtualenv/virtualenv.ini delete mode 100644 config/.curlrc delete mode 100644 config/.default-gems delete mode 100644 config/.default-npm-packages delete mode 100644 config/.default-python-packages delete mode 100644 config/.env.12-10-stable delete mode 100644 config/.env.12-2-stable delete mode 100644 config/.env.12-3-stable delete mode 100644 config/.env.12-4-stable delete mode 100644 config/.env.12-5-stable delete mode 100644 config/.env.12-6-stable delete mode 100644 config/.env.12-7-stable delete mode 100644 config/.env.12-8-stable delete mode 100644 config/.env.12-9-stable delete mode 100644 config/.env.13-0-stable delete mode 100644 config/.env.legacy delete mode 100644 config/.gemrc delete mode 100644 config/.gradle/init.gradle delete mode 100644 config/.m2/settings.xml delete mode 100644 config/.npmrc delete mode 100644 config/.profile delete mode 100644 config/.tool-versions create mode 100644 config/files/.asdfrc create mode 100644 config/files/.bashrc create mode 100644 config/files/.config/NuGet/NuGet.Config create mode 100644 config/files/.config/virtualenv/virtualenv.ini create mode 100644 config/files/.curlrc create mode 100644 config/files/.default-gems create mode 100644 config/files/.default-npm-packages create mode 100644 config/files/.default-python-packages create mode 100644 config/files/.env.12-10-stable create mode 100644 config/files/.env.12-2-stable create mode 100644 config/files/.env.12-3-stable create mode 100644 config/files/.env.12-4-stable create mode 100644 config/files/.env.12-5-stable create mode 100644 config/files/.env.12-6-stable create mode 100644 config/files/.env.12-7-stable create mode 100644 config/files/.env.12-8-stable create mode 100644 config/files/.env.12-9-stable create mode 100644 config/files/.env.13-0-stable create mode 100644 config/files/.env.legacy create mode 100644 config/files/.gemrc create mode 100644 config/files/.gradle/init.gradle create mode 100644 config/files/.m2/settings.xml create mode 100644 config/files/.npmrc create mode 100644 config/files/.profile create mode 100644 config/files/.tool-versions create mode 100644 config/projects/license_management.rb create mode 100755 config/scripts/license_management/postinst create mode 100755 config/scripts/license_management/postrm create mode 100755 config/scripts/license_management/preinst create mode 100755 config/scripts/license_management/prerm create mode 100644 config/software/license_management.rb create mode 100644 config/software/ruby.rb create mode 100755 exe/scan_bundler create mode 100644 omnibus.rb create mode 100644 pkg/.keep create mode 100644 spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile create mode 100644 spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile.lock create mode 100644 spec/support/project_helper.rb create mode 100644 spec/support/report.rb create mode 100644 spec/unit/license_finder/bundler_spec.rb diff --git a/.dockerignore b/.dockerignore index 22b59c0..26c9de0 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,8 +1,9 @@ -bin +coverage Dockerfile .dockerignore .git* +pkg spec tags tmp -config/.env* +vendor diff --git a/.gitignore b/.gitignore index 87241ef..ee6686f 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ Dockerfile.env pkg tmp coverage +vendor diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4746712..0907992 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -4,7 +4,6 @@ variables: DOCKER_DRIVER: overlay2 GIT_DEPTH: "1" GIT_STRATEGY: fetch - LATEST_IMAGE: registry.gitlab.com/gitlab-org/security-products/license-management:latest MAJOR: 3 TMP_IMAGE: $CI_REGISTRY_IMAGE/license-finder:$CI_COMMIT_SHA diff --git a/.gitlab/build.yml b/.gitlab/build.yml index a7c2dab..17005c2 100644 --- a/.gitlab/build.yml +++ b/.gitlab/build.yml @@ -22,11 +22,8 @@ build-docker-image: services: - docker:stable-dind script: - - docker info - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY - - docker pull $LATEST_IMAGE || true - - docker build --cache-from $LATEST_IMAGE -t $TMP_IMAGE . - - docker image inspect $TMP_IMAGE --format='{{.Size}}' + - IMAGE_NAME="$TMP_IMAGE" bin/docker-build - docker push $TMP_IMAGE build-mvn-pkg: diff --git a/.gitlab/test.yml b/.gitlab/test.yml index 506f344..00c9434 100644 --- a/.gitlab/test.yml +++ b/.gitlab/test.yml @@ -22,7 +22,7 @@ lint: stage: test image: ruby:alpine script: - - apk add build-base shellcheck + - apk add bash build-base git shellcheck - bin/setup - bin/lint needs: [] @@ -30,17 +30,16 @@ lint: .rspec: stage: test script: - - bash -lc './bin/test $RSPEC_DIR --format RspecJunitFormatter --out rspec.xml' + - ./bin/setup + - ./bin/test $RSPEC_DIR --format RspecJunitFormatter --out rspec.xml variables: - BUNDLE_JOBS: '2' - BUNDLE_PATH: 'vendor/bundle' GIT_DEPTH: "10" GIT_STRATEGY: fetch LOG_LEVEL: debug cache: key: ${CI_COMMIT_REF_SLUG} paths: - - vendor/bundle + - vendor artifacts: paths: - coverage/coverage.xml diff --git a/.rubocop.yml b/.rubocop.yml index 32a0c0e..deaab88 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -6,10 +6,14 @@ require: - rubocop-rspec AllCops: - TargetRubyVersion: 2.4 + TargetRubyVersion: 2.7 Exclude: - 'tmp/**/*' - 'spec/fixtures/**/*' + - 'vendor/**/*' + +Cop/GemFetcher: + Enabled: false Naming/ClassAndModuleCamelCase: Exclude: @@ -21,3 +25,6 @@ Layout/IndentFirstArrayElement: Layout/IndentFirstHashElement: EnforcedStyle: consistent + +Rails/SkipsModelValidations: + Enabled: false diff --git a/CHANGELOG.md b/CHANGELOG.md index 222529f..c135e5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # GitLab License management changelog +## v3.17.0 + +- Isolate the embedded LicenseFinder Ruby from the target project's Ruby (!181) + ## v3.16.0 - Install `dotnet` and `mono` at scan time to decrease size of Docker image. (!185) diff --git a/Dockerfile b/Dockerfile index 2c7207c..a5119c3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,38 +1,19 @@ -FROM debian:stable-slim AS gem-builder -ENV LM_HOME=/opt/license-management -WORKDIR $LM_HOME -COPY exe exe/ -COPY lib lib/ -COPY *.gemspec ./ -COPY *.json ./ -COPY *.md ./ -COPY *.yml ./ -RUN apt-get update -q \ - && apt-get install -y --no-install-recommends ruby \ - && gem build *.gemspec +# syntax = docker/dockerfile:experimental +FROM debian:stable AS deb-builder +WORKDIR /build +COPY . ./ +RUN ./bin/omnibus setup +RUN ./bin/omnibus build license_management -# Install org.codehaus.mojo:license-maven-plugin to $HOME/.m2/repository -# Install gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin to $HOME/.m2/repository -FROM debian:stable AS license-maven-plugin-builder -RUN apt-get update -q \ - && apt-get install -y --no-install-recommends maven \ - && mvn license:license-list \ - && mvn dependency:get -Dartifact=gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0 -DremoteRepositories=https://plugins.gradle.org/m2 \ - && mvn dependency:get -Dartifact=org.codehaus.plexus:plexus-utils:2.0.6 - -FROM debian:stable-slim as tools-builder +FROM debian:stable-slim ENV ASDF_DATA_DIR="/opt/asdf" -ENV HOME=/root -ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:${HOME}/.local/bin:${PATH}" +ENV PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:/opt/gitlab/.local/bin:${PATH}" ENV TERM="xterm" -WORKDIR $HOME -COPY config /root +WORKDIR /opt/gitlab COPY config/01_nodoc /etc/dpkg/dpkg.cfg.d/01_nodoc -RUN bash /root/install.sh - -FROM tools-builder -ENV LM_HOME=/opt/license-management -COPY --from=license-maven-plugin-builder /root/.m2/repository /root/.m2/repository -COPY --from=gem-builder /opt/license-management/*.gem $LM_HOME/pkg/ +RUN mkdir -p /opt/toolcache +COPY --from=deb-builder /build/pkg/*.deb /opt/toolcache/ +COPY config/install.sh /opt/install.sh +RUN bash /opt/install.sh COPY run.sh / ENTRYPOINT ["/run.sh"] diff --git a/Gemfile b/Gemfile index fa75df1..07650c5 100644 --- a/Gemfile +++ b/Gemfile @@ -1,3 +1,4 @@ source 'https://rubygems.org' gemspec +gem 'omnibus-software', git: 'https://github.com/chef/omnibus-software.git' diff --git a/Gemfile.lock b/Gemfile.lock index 1180cf2..f2a916b 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,14 @@ +GIT + remote: https://github.com/chef/omnibus-software.git + revision: 2cf96c6c07de7d05ded6b45a0531feb10ae7cd9e + specs: + omnibus-software (4.0.0) + omnibus (>= 5.6.1) + PATH remote: . specs: - license-management (3.16.0) + license-management (3.17.0) license_finder (~> 6.6.0) GEM @@ -10,28 +17,109 @@ GEM addressable (2.7.0) public_suffix (>= 2.0.2, < 5.0) ast (2.4.0) + awesome_print (1.8.0) + aws-eventstream (1.1.0) + aws-partitions (1.336.0) + aws-sdk-core (3.102.1) + aws-eventstream (~> 1, >= 1.0.2) + aws-partitions (~> 1, >= 1.239.0) + aws-sigv4 (~> 1.1) + jmespath (~> 1.0) + aws-sdk-kms (1.35.0) + aws-sdk-core (~> 3, >= 3.99.0) + aws-sigv4 (~> 1.1) + aws-sdk-s3 (1.72.0) + aws-sdk-core (~> 3, >= 3.102.1) + aws-sdk-kms (~> 1) + aws-sigv4 (~> 1.1) + aws-sigv4 (1.2.1) + aws-eventstream (~> 1, >= 1.0.2) byebug (11.1.3) + chef-cleanroom (1.0.2) + chef-config (16.2.50) + addressable + chef-utils (= 16.2.50) + fuzzyurl + mixlib-config (>= 2.2.12, < 4.0) + mixlib-shellout (>= 2.0, < 4.0) + tomlrb (~> 1.2) + chef-sugar (5.1.9) + chef-utils (16.2.50) + citrus (3.0.2) diff-lcs (1.3) docile (1.3.2) + ffi (1.13.1) + ffi-yajl (2.3.3) + libyajl2 (~> 1.2) + fuzzyurl (0.9.0) gitlab-styles (3.1.0) rubocop (~> 0.74.0) rubocop-gitlab-security (~> 0.1.0) rubocop-performance (~> 1.4.1) rubocop-rails (~> 2.0) rubocop-rspec (~> 1.36) + iostruct (0.0.4) + ipaddress (0.8.3) jaro_winkler (1.5.4) + jmespath (1.4.0) json-schema (2.8.1) addressable (>= 2.4) - license_finder (6.6.1) + libyajl2 (1.2.0) + license_finder (6.6.2) bundler rubyzip (>= 1, < 3) thor (~> 1.0.1) tomlrb (~> 1.3.0) with_env (= 1.1.0) xml-simple (~> 1.1.5) + license_scout (1.1.8) + ffi-yajl (~> 2.2) + mixlib-shellout (>= 2.2, < 4.0) + toml-rb (>= 1, < 3) + mixlib-cli (2.1.6) + mixlib-config (3.0.6) + tomlrb + mixlib-log (3.0.8) + mixlib-shellout (3.0.9) + mixlib-versioning (1.2.12) + multipart-post (2.1.1) + ohai (16.2.1) + chef-config (>= 12.8, < 17) + chef-utils (>= 16.0, < 17) + ffi (~> 1.9) + ffi-yajl (~> 2.2) + ipaddress + mixlib-cli (>= 1.7.0) + mixlib-config (>= 2.0, < 4.0) + mixlib-log (>= 2.0.1, < 4.0) + mixlib-shellout (>= 2.0, < 4.0) + plist (~> 3.1) + systemu (~> 2.6.4) + wmi-lite (~> 1.0) + omnibus (7.0.13) + aws-sdk-s3 (~> 1) + chef-cleanroom (~> 1.0) + chef-sugar (>= 3.3) + ffi-yajl (~> 2.2) + license_scout (~> 1.0) + mixlib-shellout (>= 2.0, < 4.0) + mixlib-versioning + ohai (>= 13, < 17) + pedump + ruby-progressbar (~> 1.7) + thor (>= 0.18, < 2.0) parallel (1.19.1) parser (2.7.0.4) ast (~> 2.4.0) + pedump (0.5.4) + awesome_print + iostruct (>= 0.0.4) + multipart-post (>= 2.0.0) + progressbar + rainbow + zhexdump (>= 0.0.2) + plist (3.5.0) + progressbar (1.10.1) public_suffix (4.0.3) rack (2.2.2) rainbow (3.0.0) @@ -74,11 +162,16 @@ GEM simplecov-cobertura (1.3.1) simplecov (~> 0.8) simplecov-html (0.12.2) + systemu (2.6.5) thor (1.0.1) + toml-rb (2.0.1) + citrus (~> 3.0, > 3.0) tomlrb (1.3.0) unicode-display_width (1.6.1) with_env (1.1.0) + wmi-lite (1.0.5) xml-simple (1.1.5) + zhexdump (0.0.2) PLATFORMS ruby @@ -88,6 +181,8 @@ DEPENDENCIES gitlab-styles (~> 3.1) json-schema (~> 2.8) license-management! + omnibus (~> 7.0) + omnibus-software! rspec (~> 3.9) rspec_junit_formatter (~> 0.4) simplecov (~> 0.18) diff --git a/README.md b/README.md index 930c23b..b3a1789 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ You can run the tests from inside a docker container: ```sh $ ./bin/docker-build $ ./bin/docker-shell -$ cd /opt/license-management/ +$ ./bin/setup $ ./bin/test ``` @@ -54,12 +54,11 @@ following these steps: ```sh $ ./bin/docker-build $ ./bin/docker-shell -$ cd /opt/license-management/ $ enable_dev_mode $ bundle open license_finder ``` -The `docker-shell` script will mount the current project as a volume into `/opt/license-management`. +The `docker-shell` script will mount the current project as a volume into `/builds/gitlab-org/security-products/license-management`. This allows you to edit code from your host machine using your preferred editor and see the affect of those changes from within the running docker container. diff --git a/bin/docker-build b/bin/docker-build index ba694f1..6f111fe 100755 --- a/bin/docker-build +++ b/bin/docker-build @@ -6,10 +6,10 @@ cd "$(dirname "$0")/.." LATEST_IMAGE=${LATEST_IMAGE:='registry.gitlab.com/gitlab-org/security-products/license-management:latest'} IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest} +export DOCKER_BUILDKIT=1 if command -v docker; then - docker pull $LATEST_IMAGE - docker build --network=host --cache-from "$LATEST_IMAGE" -t "$IMAGE_NAME" . + docker build --progress=plain --network=host --cache-from "$LATEST_IMAGE" -t "$IMAGE_NAME" . else echo "Install docker: https://docs.docker.com/engine/installation/" exit 1 diff --git a/bin/docker-shell b/bin/docker-shell index 617492a..c6c03a0 100755 --- a/bin/docker-shell +++ b/bin/docker-shell @@ -9,5 +9,5 @@ IMAGE_NAME=${IMAGE_NAME:-$(basename "$PWD"):latest} docker run --rm -it \ --entrypoint='' \ --network=host \ - --volume "$PWD":/opt/license-management \ - "$IMAGE_NAME" /bin/bash -l + --volume "$PWD":/builds/gitlab-org/security-products/license-management \ + "$IMAGE_NAME" sh -c 'cd /builds/gitlab-org/security-products/license-management && exec bash -l' diff --git a/bin/lint b/bin/lint index 97cb1c3..dada38c 100755 --- a/bin/lint +++ b/bin/lint @@ -5,8 +5,8 @@ set -e cd "$(dirname "$0")/.." shellcheck bin/* -shellcheck config/.bashrc -shellcheck config/.profile +shellcheck config/files/.bashrc +shellcheck config/files/.profile shellcheck config/*.sh shellcheck run.sh bundle exec rubocop diff --git a/bin/omnibus b/bin/omnibus new file mode 100755 index 0000000..526b8d7 --- /dev/null +++ b/bin/omnibus @@ -0,0 +1,46 @@ +#!/bin/sh + +set -e + +cd "$(dirname "$0")/.." + +case $1 in + setup) + apt-get update -q + apt-get install -y --no-install-recommends \ + autoconf \ + automake \ + bison \ + build-essential \ + ca-certificates \ + cmake \ + curl \ + fakeroot \ + gettext \ + git \ + libdb-dev \ + libffi-dev \ + libgdbm-dev \ + libgdbm6 \ + libncurses5-dev \ + libreadline6-dev \ + libssl-dev \ + libtool \ + libtool-bin \ + libyaml-dev \ + maven \ + python3 \ + ruby \ + ruby-dev \ + zlib1g-dev \ + zstd + + gem install bundler + bundle install + ;; + + build) + for i in "$@"; do :; done + bundle exec omnibus build -l debug "$i" + ;; +esac diff --git a/bin/setup b/bin/setup index b54a99f..03f0938 100755 --- a/bin/setup +++ b/bin/setup @@ -1,11 +1,16 @@ -#!/bin/sh +#!/bin/bash -l set -e cd "$(dirname "$0")/.." -gem install bundler --conservative -v '~> 2.0' -q -bundle install --quiet +export PATH="/builds/gitlab-org/security-products/license-management/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + if [ ! -f /usr/sbin/haproxy ] && command -v apt-get; then apt-get update -y && apt-get install -y --no-install-recommends haproxy fi + +[[ -z "$CI_JOB_ID" ]] && enable_dev_mode +bundle config --local path vendor +bundle config --local jobs "$(nproc)" +bundle install diff --git a/bin/test b/bin/test index 218224d..a05c7e7 100755 --- a/bin/test +++ b/bin/test @@ -1,8 +1,13 @@ -#!/bin/sh +#!/bin/bash -l set -e cd "$(dirname "$0")/.." -./bin/setup +export PATH="/builds/gitlab-org/security-products/license-management/exe:/opt/gitlab/embedded/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +export RUBYLIB="/builds/gitlab-org/security-products/license-management/lib" + +if ! command -v bundle; then + ./bin/setup +fi bundle exec rspec "$@" --format=progress --tag ~environment:offline diff --git a/config/.asdfrc b/config/.asdfrc deleted file mode 100644 index e600bd0..0000000 --- a/config/.asdfrc +++ /dev/null @@ -1 +0,0 @@ -legacy_version_file = yes diff --git a/config/.bashrc b/config/.bashrc deleted file mode 100644 index 3e45713..0000000 --- a/config/.bashrc +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -alias nuget='mono /usr/local/bin/nuget.exe' -set -o vi - -function inflate() { - local file=$1 - local to_dir=$2 - if [ -f "$file" ]; then - tar --use-compress-program zstd -xf "$file" -C "$to_dir" - rm "$file" - fi -} - -update_java_home() { - local java_path - java_path="$(asdf which java)" - if [[ -n "${java_path}" ]]; then - export JAVA_HOME - JAVA_HOME="$(dirname "$(dirname "$(realpath "${java_path}")")")" - fi -} - -function switch_to_exact() { - local tool=$1 - local version=$2 - - asdf shell "$tool" "$version" - if [[ "$tool" = "java" ]]; then - update_java_home - fi -} - -function switch_to() { - local tool=$1 - local major_version=$2 - local version - version="$(grep "$tool" "$HOME/.tool-versions"| tr ' ' '\n' | grep "^$major_version")" - - switch_to_exact "$tool" "$version" -} - -function major_version_from() { - echo "$1" | cut -d'.' -f1 -} - -function enable_dev_mode() { - unset HISTFILESIZE - unset HISTSIZE - export EDITOR=vim - export LOG_LEVEL=debug - set -o vi - apt-get update -y - apt-get install -y --no-install-recommends vim less shellcheck -} - -inflate /opt/asdf.tar.zst /opt -inflate /usr/lib/gcc.tar.zst /usr/lib -inflate /usr/lib/rustlib.tar.zst /usr/lib - -# shellcheck source=/dev/null -. "$ASDF_DATA_DIR/asdf.sh" -# shellcheck source=/dev/null -. "$ASDF_DATA_DIR/completions/asdf.bash" diff --git a/config/.config/NuGet/NuGet.Config b/config/.config/NuGet/NuGet.Config deleted file mode 100644 index 449478c..0000000 --- a/config/.config/NuGet/NuGet.Config +++ /dev/null @@ -1,13 +0,0 @@ - - - - - - - - - - - - - diff --git a/config/.config/virtualenv/virtualenv.ini b/config/.config/virtualenv/virtualenv.ini deleted file mode 100644 index 208c7bf..0000000 --- a/config/.config/virtualenv/virtualenv.ini +++ /dev/null @@ -1,4 +0,0 @@ -[virtualenv] -python = /opt/asdf/shims/python -activators = - bash diff --git a/config/.curlrc b/config/.curlrc deleted file mode 100644 index df02a54..0000000 --- a/config/.curlrc +++ /dev/null @@ -1,2 +0,0 @@ -# Don't show progress meter ---silent diff --git a/config/.default-gems b/config/.default-gems deleted file mode 100644 index 87e0d78..0000000 --- a/config/.default-gems +++ /dev/null @@ -1,3 +0,0 @@ -bundler ~>1.7 -bundler ~>2.0 -license_finder ~>6.6.0 diff --git a/config/.default-npm-packages b/config/.default-npm-packages deleted file mode 100644 index 8e16e61..0000000 --- a/config/.default-npm-packages +++ /dev/null @@ -1,4 +0,0 @@ -bower -bower-npm-resolver -npm-install-peers -yarn diff --git a/config/.default-python-packages b/config/.default-python-packages deleted file mode 100644 index ddf6574..0000000 --- a/config/.default-python-packages +++ /dev/null @@ -1,4 +0,0 @@ -conan -pip -pipenv -virtualenv diff --git a/config/.env.12-10-stable b/config/.env.12-10-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.12-10-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.12-2-stable b/config/.env.12-2-stable deleted file mode 100644 index 808b2ac..0000000 --- a/config/.env.12-2-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION 1 diff --git a/config/.env.12-3-stable b/config/.env.12-3-stable deleted file mode 100644 index 808b2ac..0000000 --- a/config/.env.12-3-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION 1 diff --git a/config/.env.12-4-stable b/config/.env.12-4-stable deleted file mode 100644 index 86a9c8f..0000000 --- a/config/.env.12-4-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '1.1' diff --git a/config/.env.12-5-stable b/config/.env.12-5-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.12-5-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.12-6-stable b/config/.env.12-6-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.12-6-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.12-7-stable b/config/.env.12-7-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.12-7-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.12-8-stable b/config/.env.12-8-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.12-8-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.12-9-stable b/config/.env.12-9-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.12-9-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.13-0-stable b/config/.env.13-0-stable deleted file mode 100644 index da5a244..0000000 --- a/config/.env.13-0-stable +++ /dev/null @@ -1,2 +0,0 @@ -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION '2.0' diff --git a/config/.env.legacy b/config/.env.legacy deleted file mode 100644 index 7181ab8..0000000 --- a/config/.env.legacy +++ /dev/null @@ -1,3 +0,0 @@ -LM_PYTHON_VERSION 2.7 -LM_REPORT_FILE gl-license-management-report.json -LM_REPORT_VERSION 1 diff --git a/config/.gemrc b/config/.gemrc deleted file mode 100644 index ff5fcc7..0000000 --- a/config/.gemrc +++ /dev/null @@ -1,4 +0,0 @@ -:verbose: true -:sources: -- https://rubygems.org/ -gem: --no-document diff --git a/config/.gradle/init.gradle b/config/.gradle/init.gradle deleted file mode 100644 index b8cc7f1..0000000 --- a/config/.gradle/init.gradle +++ /dev/null @@ -1,13 +0,0 @@ -initscript { - repositories { - maven { url uri('/root/.m2/repository') } - maven { url "https://plugins.gradle.org/m2" } - } - dependencies { - classpath "gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0" - } -} -allprojects { - project.apply plugin: com.hierynomus.gradle.license.LicenseBasePlugin - project.apply plugin: com.hierynomus.gradle.license.LicenseReportingPlugin -} diff --git a/config/.m2/settings.xml b/config/.m2/settings.xml deleted file mode 100644 index 9a89d90..0000000 --- a/config/.m2/settings.xml +++ /dev/null @@ -1,32 +0,0 @@ - - ${user.home}/.m2/repository - false - false - - - gitlab-maven - - true - - - - gitlab-maven - ${env.CI_API_V4_URL}/projects/${env.CI_PROJECT_ID}/packages/maven - - - - - - - gitlab-maven - - - - Job-Token - ${env.CI_JOB_TOKEN} - - - - - - diff --git a/config/.npmrc b/config/.npmrc deleted file mode 100644 index 426026b..0000000 --- a/config/.npmrc +++ /dev/null @@ -1,13 +0,0 @@ -audit=false -dev=false -fetch-retries=2 -fetch-retry-max-timeout=10000 -fetch-retry-min-timeout=5000 -ignore-scripts=true -log-level=silly -only=production -optional=true -package-lock=true -prefer-offline=true -progress=false -send-metrics=false diff --git a/config/.profile b/config/.profile deleted file mode 100644 index 320c173..0000000 --- a/config/.profile +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/sh -# shellcheck source=/dev/null -. "$HOME/.bashrc" diff --git a/config/.tool-versions b/config/.tool-versions deleted file mode 100644 index 852c2d4..0000000 --- a/config/.tool-versions +++ /dev/null @@ -1,11 +0,0 @@ -elixir system -golang 1.14 -gradle 6.3 -java adoptopenjdk-8.0.252+9.1 adoptopenjdk-11.0.7+10.1 -maven 3.6.3 -nodejs 12.16.3 10.20.1 -php 7.4.5 -python 3.8.2 2.7.18 -ruby 2.6.6 -rust system -sbt 1.3.8 diff --git a/config/files/.asdfrc b/config/files/.asdfrc new file mode 100644 index 0000000..e600bd0 --- /dev/null +++ b/config/files/.asdfrc @@ -0,0 +1 @@ +legacy_version_file = yes diff --git a/config/files/.bashrc b/config/files/.bashrc new file mode 100644 index 0000000..81afcb8 --- /dev/null +++ b/config/files/.bashrc @@ -0,0 +1,69 @@ +#!/bin/bash + +export ASDF_DATA_DIR="/opt/asdf" +export PATH="${ASDF_DATA_DIR}/shims:${ASDF_DATA_DIR}/bin:/opt/gitlab/.local/bin:${PATH}" +export HOME="/opt/gitlab" + +alias nuget='mono /usr/local/bin/nuget.exe' +set -o vi + + +function inflate() { + local file=$1 + local to_dir=$2 + if [ -f "$file" ]; then + tar --use-compress-program zstd -xf "$file" -C "$to_dir" + rm "$file" + fi +} + +update_java_home() { + local java_path + java_path="$(asdf which java)" + if [[ -n "${java_path}" ]]; then + export JAVA_HOME + JAVA_HOME="$(dirname "$(dirname "$(realpath "${java_path}")")")" + fi +} + +function switch_to_exact() { + local tool=$1 + local version=$2 + + asdf shell "$tool" "$version" + if [[ "$tool" = "java" ]]; then + update_java_home + fi +} + +function switch_to() { + local tool=$1 + local major_version=$2 + local version + version="$(grep "$tool" "/opt/gitlab/.tool-versions"| tr ' ' '\n' | grep "^$major_version")" + + switch_to_exact "$tool" "$version" +} + +function major_version_from() { + echo "$1" | cut -d'.' -f1 +} + +function enable_dev_mode() { + unset HISTFILESIZE + unset HISTSIZE + export EDITOR=vim + export LOG_LEVEL=debug + set -o vi + apt-get update -y + apt-get install -y --no-install-recommends vim less shellcheck +} + +inflate /opt/asdf.tar.zst /opt +inflate /usr/lib/gcc.tar.zst /usr/lib +inflate /usr/lib/rustlib.tar.zst /usr/lib + +# shellcheck source=/dev/null +. "$ASDF_DATA_DIR/asdf.sh" +# shellcheck source=/dev/null +. "$ASDF_DATA_DIR/completions/asdf.bash" diff --git a/config/files/.config/NuGet/NuGet.Config b/config/files/.config/NuGet/NuGet.Config new file mode 100644 index 0000000..449478c --- /dev/null +++ b/config/files/.config/NuGet/NuGet.Config @@ -0,0 +1,13 @@ + + + + + + + + + + + + + diff --git a/config/files/.config/virtualenv/virtualenv.ini b/config/files/.config/virtualenv/virtualenv.ini new file mode 100644 index 0000000..208c7bf --- /dev/null +++ b/config/files/.config/virtualenv/virtualenv.ini @@ -0,0 +1,4 @@ +[virtualenv] +python = /opt/asdf/shims/python +activators = + bash diff --git a/config/files/.curlrc b/config/files/.curlrc new file mode 100644 index 0000000..df02a54 --- /dev/null +++ b/config/files/.curlrc @@ -0,0 +1,2 @@ +# Don't show progress meter +--silent diff --git a/config/files/.default-gems b/config/files/.default-gems new file mode 100644 index 0000000..f832f6e --- /dev/null +++ b/config/files/.default-gems @@ -0,0 +1,2 @@ +bundler ~>1.7 +bundler ~>2.0 diff --git a/config/files/.default-npm-packages b/config/files/.default-npm-packages new file mode 100644 index 0000000..8e16e61 --- /dev/null +++ b/config/files/.default-npm-packages @@ -0,0 +1,4 @@ +bower +bower-npm-resolver +npm-install-peers +yarn diff --git a/config/files/.default-python-packages b/config/files/.default-python-packages new file mode 100644 index 0000000..ddf6574 --- /dev/null +++ b/config/files/.default-python-packages @@ -0,0 +1,4 @@ +conan +pip +pipenv +virtualenv diff --git a/config/files/.env.12-10-stable b/config/files/.env.12-10-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.12-10-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.12-2-stable b/config/files/.env.12-2-stable new file mode 100644 index 0000000..808b2ac --- /dev/null +++ b/config/files/.env.12-2-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION 1 diff --git a/config/files/.env.12-3-stable b/config/files/.env.12-3-stable new file mode 100644 index 0000000..808b2ac --- /dev/null +++ b/config/files/.env.12-3-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION 1 diff --git a/config/files/.env.12-4-stable b/config/files/.env.12-4-stable new file mode 100644 index 0000000..86a9c8f --- /dev/null +++ b/config/files/.env.12-4-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '1.1' diff --git a/config/files/.env.12-5-stable b/config/files/.env.12-5-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.12-5-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.12-6-stable b/config/files/.env.12-6-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.12-6-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.12-7-stable b/config/files/.env.12-7-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.12-7-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.12-8-stable b/config/files/.env.12-8-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.12-8-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.12-9-stable b/config/files/.env.12-9-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.12-9-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.13-0-stable b/config/files/.env.13-0-stable new file mode 100644 index 0000000..da5a244 --- /dev/null +++ b/config/files/.env.13-0-stable @@ -0,0 +1,2 @@ +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION '2.0' diff --git a/config/files/.env.legacy b/config/files/.env.legacy new file mode 100644 index 0000000..7181ab8 --- /dev/null +++ b/config/files/.env.legacy @@ -0,0 +1,3 @@ +LM_PYTHON_VERSION 2.7 +LM_REPORT_FILE gl-license-management-report.json +LM_REPORT_VERSION 1 diff --git a/config/files/.gemrc b/config/files/.gemrc new file mode 100644 index 0000000..8bfd7f0 --- /dev/null +++ b/config/files/.gemrc @@ -0,0 +1,4 @@ +backtrace: true +benchmark: false +gem: --no-ri --no-rdoc --no-document --suggestions +verbose: true diff --git a/config/files/.gradle/init.gradle b/config/files/.gradle/init.gradle new file mode 100644 index 0000000..b8cc7f1 --- /dev/null +++ b/config/files/.gradle/init.gradle @@ -0,0 +1,13 @@ +initscript { + repositories { + maven { url uri('/root/.m2/repository') } + maven { url "https://plugins.gradle.org/m2" } + } + dependencies { + classpath "gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0" + } +} +allprojects { + project.apply plugin: com.hierynomus.gradle.license.LicenseBasePlugin + project.apply plugin: com.hierynomus.gradle.license.LicenseReportingPlugin +} diff --git a/config/files/.m2/settings.xml b/config/files/.m2/settings.xml new file mode 100644 index 0000000..9a89d90 --- /dev/null +++ b/config/files/.m2/settings.xml @@ -0,0 +1,32 @@ + + ${user.home}/.m2/repository + false + false + + + gitlab-maven + + true + + + + gitlab-maven + ${env.CI_API_V4_URL}/projects/${env.CI_PROJECT_ID}/packages/maven + + + + + + + gitlab-maven + + + + Job-Token + ${env.CI_JOB_TOKEN} + + + + + + diff --git a/config/files/.npmrc b/config/files/.npmrc new file mode 100644 index 0000000..426026b --- /dev/null +++ b/config/files/.npmrc @@ -0,0 +1,13 @@ +audit=false +dev=false +fetch-retries=2 +fetch-retry-max-timeout=10000 +fetch-retry-min-timeout=5000 +ignore-scripts=true +log-level=silly +only=production +optional=true +package-lock=true +prefer-offline=true +progress=false +send-metrics=false diff --git a/config/files/.profile b/config/files/.profile new file mode 100644 index 0000000..f7638e6 --- /dev/null +++ b/config/files/.profile @@ -0,0 +1,3 @@ +#!/bin/sh +# shellcheck source=/dev/null +. "/opt/gitlab/.bashrc" diff --git a/config/files/.tool-versions b/config/files/.tool-versions new file mode 100644 index 0000000..852c2d4 --- /dev/null +++ b/config/files/.tool-versions @@ -0,0 +1,11 @@ +elixir system +golang 1.14 +gradle 6.3 +java adoptopenjdk-8.0.252+9.1 adoptopenjdk-11.0.7+10.1 +maven 3.6.3 +nodejs 12.16.3 10.20.1 +php 7.4.5 +python 3.8.2 2.7.18 +ruby 2.6.6 +rust system +sbt 1.3.8 diff --git a/config/install.sh b/config/install.sh index a93451b..0de9b1c 100644 --- a/config/install.sh +++ b/config/install.sh @@ -81,10 +81,14 @@ wget -q -O /etc/apt/sources.list.d/microsoft-prod.list https://packages.microsof apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 3FA7E0328081BFF6A14DA29AA6A19B38D3D831EF echo "deb https://download.mono-project.com/repo/debian stable-buster main" | tee /etc/apt/sources.list.d/mono-official-stable.list -curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe & +curl -o /usr/local/bin/nuget.exe https://dist.nuget.org/win-x86-commandline/latest/nuget.exe echo -e "section_end:$(date +%s):install_dotnet\r\e[0K" echo -e "section_start:$(date +%s):install_asdf\r\e[0K==> Installing asdf…" +dpkg --install "$(find /opt/toolcache/ -name "license-management*.deb")" +rm -fr /root +ln -s /opt/gitlab /root + mkdir -p "$ASDF_DATA_DIR" git clone https://github.com/asdf-vm/asdf.git "$ASDF_DATA_DIR" cd "$ASDF_DATA_DIR" @@ -96,7 +100,7 @@ git checkout "$(git describe --abbrev=0 --tags)" while IFS= read -r line; do tool=$(echo "$line" | cut -d' ' -f1) asdf plugin-add "$tool" -done < "$HOME/.tool-versions" +done < "/opt/gitlab/.tool-versions" bash "$ASDF_DATA_DIR/plugins/nodejs/bin/import-release-team-keyring" asdf install asdf reshim @@ -104,7 +108,7 @@ asdf current for version in $(asdf list python); do asdf shell python "$version" - pip download -d "$HOME/.config/virtualenv/app-data" pip-licenses pip setuptools wheel + pip download -d "/opt/gitlab/.config/virtualenv/app-data" pip-licenses pip setuptools wheel done wait echo -e "section_end:$(date +%s):install_asdf\r\e[0K" @@ -127,11 +131,11 @@ rm -fr "$ASDF_DATA_DIR/docs" \ "$ASDF_DATA_DIR"/installs/ruby/**/lib/ruby/gems/**/cache \ "$ASDF_DATA_DIR"/installs/**/**/share \ "$ASDF_DATA_DIR"/test \ - "$HOME"/.config/configstore/update-notifier-npm.json \ - "$HOME"/.config/pip/selfcheck.json \ - "$HOME"/.gem \ - "$HOME"/.npm \ - "$HOME"/.wget-hsts \ + /opt/gitlab/.config/configstore/update-notifier-npm.json \ + /opt/gitlab/.config/pip/selfcheck.json \ + /opt/gitlab/.gem \ + /opt/gitlab/.npm \ + /opt/gitlab/.wget-hsts \ /etc/apache2/* \ /etc/bash_completion.d/* \ /etc/calendar/* \ @@ -186,7 +190,5 @@ wait rm -fr \ /opt/asdf/ \ /usr/lib/gcc \ - /usr/lib/mono \ - /usr/lib/rustlib \ - /usr/share/dotnet + /usr/lib/rustlib echo -e "section_end:$(date +%s):compress_files\r\e[0K" diff --git a/config/projects/license_management.rb b/config/projects/license_management.rb new file mode 100644 index 0000000..c34a680 --- /dev/null +++ b/config/projects/license_management.rb @@ -0,0 +1,17 @@ +require_relative '../../lib/license/management/version.rb' +name "license-management" +maintainer "mkhan@gitlab.com" +homepage "https://gitlab.com/gitlab-org/security-products/license-management" +license_file "LICENSE" + +install_dir "#{default_root}/gitlab" +build_version License::Management::VERSION +build_iteration 1 + +dependency "license_management" +package_scripts_path Pathname.pwd.join("config/scripts/license_management") + +package :deb do + compression_level 9 + compression_type :xz +end diff --git a/config/scripts/license_management/postinst b/config/scripts/license_management/postinst new file mode 100755 index 0000000..da63d3b --- /dev/null +++ b/config/scripts/license_management/postinst @@ -0,0 +1,14 @@ +#!/bin/sh + +PROGNAME=$(basename "$0") + +error_exit() +{ + echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +ln -s /opt/gitlab/bin/license_management /usr/local/bin/license_management +echo "Thank you for installing license_management!" + +exit 0 diff --git a/config/scripts/license_management/postrm b/config/scripts/license_management/postrm new file mode 100755 index 0000000..5b47b27 --- /dev/null +++ b/config/scripts/license_management/postrm @@ -0,0 +1,6 @@ +#!/bin/sh + +rm -f /usr/local/bin/license_management +echo "license_management has been uninstalled!" + +exit 0 diff --git a/config/scripts/license_management/preinst b/config/scripts/license_management/preinst new file mode 100755 index 0000000..dcef0c4 --- /dev/null +++ b/config/scripts/license_management/preinst @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "You're about to install license_management!" diff --git a/config/scripts/license_management/prerm b/config/scripts/license_management/prerm new file mode 100755 index 0000000..ef80795 --- /dev/null +++ b/config/scripts/license_management/prerm @@ -0,0 +1,11 @@ +#!/bin/sh + +PROGNAME=$(basename "$0") + +error_exit() +{ + echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2 + exit 1 +} + +exit 0 diff --git a/config/software/license_management.rb b/config/software/license_management.rb new file mode 100644 index 0000000..be19ddb --- /dev/null +++ b/config/software/license_management.rb @@ -0,0 +1,34 @@ +require_relative '../../lib/license/management/version.rb' +name "license_management" + +default_version License::Management::VERSION +source({ + path: Dir.pwd, + options: { + exclude: %w[bin coverage pkg spec tmp vendor .git .gitlab] + } +}) + +dependency "ruby" + +build do + touch "#{install_dir}/bin/.keep" + touch "#{install_dir}/.local/bin/.keep" + + gem "build #{project_dir}/license-management.gemspec -o #{project_dir}/release.gem" + gem "install -f #{project_dir}/release.gem --no-document" + delete "#{project_dir}/release.gem" + command "cp #{install_dir}/embedded/bin/license_management #{install_dir}/bin/license_management" + + command "cp -r #{project_dir}/config/files/. #{install_dir}/" + command "mvn license:license-list" + command "mvn dependency:get -Dartifact=gradle.plugin.com.hierynomus.gradle.plugins:license-gradle-plugin:0.15.0 -DremoteRepositories=https://plugins.gradle.org/m2" + command "mvn dependency:get -Dartifact=org.codehaus.plexus:plexus-utils:2.0.6" + copy "#{Dir.home}/.m2/repository/", "#{install_dir}/.m2/repository/" +end + +build do + delete "#{install_dir}/embedded/lib/ruby/gems/**/cache" + delete "#{install_dir}/embedded/lib/ruby/gems/**/doc" + delete "#{install_dir}/embedded/lib/ruby/gems/**/build_info" +end diff --git a/config/software/ruby.rb b/config/software/ruby.rb new file mode 100644 index 0000000..56a04fc --- /dev/null +++ b/config/software/ruby.rb @@ -0,0 +1,73 @@ +require 'fileutils' + +name "ruby" + +license "BSD-2-Clause" +license_file "BSDL" +license_file "COPYING" +license_file "LEGAL" +skip_transitive_dependency_licensing true + +default_version "2.7.1" + +dependency "libffi" +dependency "libyaml" +dependency "openssl" +dependency "zlib" + +source url: "https://cache.ruby-lang.org/pub/ruby/#{version.match(/^(\d+\.\d+)/)[0]}/ruby-#{version}.tar.gz", sha256: "d418483bdd0000576c1370571121a6eb24582116db0b7bb2005e90e250eae418" + +relative_path "ruby-#{version}" +embedded_dir = "#{install_dir}/embedded" + +build do + env = with_standard_compiler_flags(with_embedded_path) + configure_command = [ + "--disable-debug-env", + "--disable-dtrace", + "--disable-install-capi", + "--disable-install-doc", + "--disable-install-rdoc", + "--disable-jit-support", + "--enable-shared", + "--prefix=#{embedded_dir}", + "--with-out-ext=dbm,readline,rdoc,win32,win32ole,sdbm", + "--without-gdbm", + "--without-gmp", + "--without-jemalloc", + "--without-tk", + "--without-valgrind" + ] + configure(*configure_command, env: env) + make "-j #{workers}", env: env + make "-j #{workers} install", env: env +end + +build do + command "#{embedded_dir}/bin/gem install bundler -v'~> 2.0' --no-document" + command "#{embedded_dir}/bin/gem install bundler -v'~> 1.0' --no-document" +end + +build do + %w[benchmark pstore rss test-unit webrick minitest].each do |gem_name| + delete "#{embedded_dir}/lib/ruby/gems/**/gems/#{gem_name}*" + delete "#{embedded_dir}/lib/ruby/**/#{gem_name}*" + end + + delete "#{embedded_dir}/bin/*rac*" + delete "#{embedded_dir}/bin/erb" + delete "#{embedded_dir}/bin/irb" + delete "#{embedded_dir}/bin/rake" + delete "#{embedded_dir}/bin/rdoc" + delete "#{embedded_dir}/bin/ri" + delete "#{embedded_dir}/bin/thor" + delete "#{embedded_dir}/lib/ruby/gems/**/cache" + delete "#{embedded_dir}/lib/ruby/gems/**/doc" + delete "#{embedded_dir}/lib/ruby/gems/**/gems/**/.github" + delete "#{embedded_dir}/lib/ruby/gems/**/gems/**/.gitignore" + delete "#{embedded_dir}/lib/ruby/gems/**/gems/**/.travis.yml" + delete "#{embedded_dir}/lib/ruby/gems/**/gems/**/bundler/man" + delete "#{embedded_dir}/lib/ruby/gems/**/gems/**/man" + delete "#{embedded_dir}/lib/ruby/gems/**/gems/**/test" + delete "#{embedded_dir}/share" +end diff --git a/exe/scan_bundler b/exe/scan_bundler new file mode 100755 index 0000000..a5b792e --- /dev/null +++ b/exe/scan_bundler @@ -0,0 +1,24 @@ +#!/opt/asdf/shims/ruby +# frozen_string_literal: true + +require 'json' +require 'bundler' +require 'pathname' + +gemfile = ARGV[0] +lockfile = ARGV[1] + +Gem.clear_paths +Bundler.reset! +Bundler.configure +Bundler::Definition.build(gemfile, lockfile, nil).specs.each do |gem| + puts JSON.generate({ + name: gem.name, + version: gem.version.to_s, + description: gem.description, + homepage: gem.homepage, + full_gem_path: gem.full_gem_path, + licenses: gem.licenses, + summary: gem.summary + }) +end diff --git a/lib/license/finder/ext/bundler.rb b/lib/license/finder/ext/bundler.rb index ddf30f5..d362be3 100644 --- a/lib/license/finder/ext/bundler.rb +++ b/lib/license/finder/ext/bundler.rb @@ -6,28 +6,21 @@ module LicenseFinder vendor_path = Pathname.pwd.join('.gitlab', 'cache', 'vendor') shell.execute([:mkdir, '-p', vendor_path.to_s]) - Dir.chdir(project_path) do - with_clean_bundler do - shell.execute([:bundle, :config, '--local', :path, vendor_path.to_s]) - shell.execute([:bundle, :install, '--verbose'], env: default_env) - end + with_clean_bundler do + _stdout, _stderr, status = shell.execute([:asdf, :current, :ruby], env: default_env) + shell.execute([:asdf, :install], env: default_env) unless status.success? + shell.execute([:bundle, :config, '--local', :path, vendor_path.to_s], env: default_env) + shell.execute([:bundle, :install, '--verbose'], env: default_env) end end def current_packages - Dir.chdir(project_path) do - gem_details.map do |gem| - Dependency.new( - 'Bundler', - gem.name, - gem.version.to_s, - description: gem.description, - detection_path: lockfile_path, - homepage: gem.homepage, - install_path: gem.full_gem_path || '/dev/null', - spec_licenses: gem.licenses, - summary: gem.summary - ) + with_clean_bundler do + stdout, _stderr, status = shell.execute(scan_command, env: default_env) + return super unless status.success? + + stdout.each_line.map do |line| + map_from(JSON.parse(line, symbolize_names: true)) end end end @@ -48,27 +41,39 @@ module LicenseFinder return custom_gemfile.basename.to_s if custom_gemfile.exist? end - if project_path.join("gems.rb").exist? - "gems.rb" - else - "Gemfile" - end + project_path.join("gems.rb").exist? ? "gems.rb" : "Gemfile" end def lockfile gemfile == 'gems.rb' ? 'gems.locked' : "#{gemfile}.lock" end + def lockfile_path + project_path.join(lockfile) + end + + def scan_command + [ + '/opt/asdf/shims/ruby', + '-W0', + ::License::Management.root.join('exe', 'scan_bundler').to_s, + detected_package_path.to_s, + lockfile_path.to_s + ] + end + def default_env @default_env ||= { 'BUNDLE_ALLOW_OFFLINE_INSTALL' => 'true', - 'BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES' => 'true', 'BUNDLE_DISABLE_VERSION_CHECK' => 'true', 'BUNDLE_ERROR_ON_STDERR' => 'true', + 'BUNDLE_GEMFILE' => "#{project_path}/#{gemfile}", 'BUNDLE_IGNORE_MESSAGES' => 'true', 'BUNDLE_JOBS' => ENV.fetch('BUNDLE_JOBS', `nproc`.chomp), 'BUNDLE_SILENCE_ROOT_WARNING' => 'true', - 'BUNDLE_WITH' => ENV.fetch('BUNDLE_WITH', included_groups.to_a.join(':')) + 'BUNDLE_SUPPRESS_INSTALL_USING_MESSAGES' => 'true', + 'BUNDLE_WITHOUT' => ENV.fetch('BUNDLE_WITHOUT', ignored_groups.to_a.join(':')), + 'PATH' => "/opt/asdf/shims:/opt/asdf/bin:#{ENV['PATH']}" }.tap do |env| env['BUNDLE_FROZEN'] = 'true' if lockfile_path.exist? env['BUNDLE_SSL_CA_CERT'] = shell.custom_certificate_path.to_s if shell.custom_certificate_installed? @@ -76,11 +81,30 @@ module LicenseFinder end def with_clean_bundler - if ::Bundler.respond_to?(:with_unbundled_env) - ::Bundler.with_unbundled_env { yield } - else - ::Bundler.with_clean_env { yield } + ::Gem.clear_paths + ::Bundler.reset! + ::Bundler.configure + Dir.chdir(project_path) do + if ::Bundler.respond_to?(:with_unbundled_env) + ::Bundler.with_unbundled_env { yield } + else + ::Bundler.with_clean_env { yield } + end end end + + def map_from(gem) + Dependency.new( + 'Bundler', + gem[:name], + gem[:version], + description: gem[:description], + detection_path: lockfile_path, + homepage: gem[:homepage], + install_path: gem[:full_gem_path] || '/dev/null', + spec_licenses: gem[:licenses], + summary: gem[:summary] + ) + end end end diff --git a/lib/license/finder/ext/package_manager.rb b/lib/license/finder/ext/package_manager.rb index ff5466e..6752ce7 100644 --- a/lib/license/finder/ext/package_manager.rb +++ b/lib/license/finder/ext/package_manager.rb @@ -2,6 +2,10 @@ module LicenseFinder class PackageManager + def shell + ::License::Management.shell + end + def current_packages_with_relations current_packages rescue StandardError => e diff --git a/lib/license/finder/ext/shared_helpers.rb b/lib/license/finder/ext/shared_helpers.rb index c3d6319..856c239 100644 --- a/lib/license/finder/ext/shared_helpers.rb +++ b/lib/license/finder/ext/shared_helpers.rb @@ -2,10 +2,6 @@ module LicenseFinder module SharedHelpers - def shell - ::License::Management.shell - end - class Cmd def self.run(command) ::License::Management.shell.execute(command) diff --git a/lib/license/management/python.rb b/lib/license/management/python.rb index ccf53e9..740b5c4 100644 --- a/lib/license/management/python.rb +++ b/lib/license/management/python.rb @@ -29,7 +29,7 @@ module License ". #{venv}/bin/activate &&", :pip, :install, '--no-index', - '--find-links $HOME/.config/virtualenv/app-data', 'pip-licenses', '&&', + '--find-links /opt/gitlab/.config/virtualenv/app-data', 'pip-licenses', '&&', 'pip-licenses', '--ignore-packages prettytable', '--with-description', diff --git a/lib/license/management/version.rb b/lib/license/management/version.rb index 214f7b5..df57113 100644 --- a/lib/license/management/version.rb +++ b/lib/license/management/version.rb @@ -2,6 +2,6 @@ module License module Management - VERSION = '3.16.0' + VERSION = '3.17.0' end end diff --git a/license-management.gemspec b/license-management.gemspec index 62a08b1..358dd0d 100644 --- a/license-management.gemspec +++ b/license-management.gemspec @@ -31,6 +31,7 @@ Gem::Specification.new do |spec| spec.add_development_dependency 'byebug', '~> 11.1' spec.add_development_dependency 'gitlab-styles', '~> 3.1' spec.add_development_dependency 'json-schema', '~> 2.8' + spec.add_development_dependency 'omnibus', '~> 7.0' spec.add_development_dependency 'rspec', '~> 3.9' spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4' spec.add_development_dependency 'simplecov', '~> 0.18' diff --git a/omnibus.rb b/omnibus.rb new file mode 100644 index 0000000..ecd6816 --- /dev/null +++ b/omnibus.rb @@ -0,0 +1,9 @@ +base_dir './tmp' +cache_dir './tmp/omnibus/cache' +git_cache_dir './tmp/omnibus/cache/git_cache' +source_dir './tmp/omnibus/src' +build_dir './tmp/omnibus/build' +package_dir './tmp/omnibus/pkg' +package_tmp './tmp/omnibus/pkg-tmp' +append_timestamp false +software_gems ['omnibus-software'] diff --git a/pkg/.keep b/pkg/.keep new file mode 100644 index 0000000..e69de29 diff --git a/run.sh b/run.sh index 738e81e..36231ea 100755 --- a/run.sh +++ b/run.sh @@ -9,7 +9,7 @@ export CI_API_V4_URL="${CI_API_V4_URL:-https://gitlab.com/api/v4}" export CI_DEBUG_TRACE=${CI_DEBUG_TRACE:='false'} export DOTNET_CLI_TELEMETRY_OPTOUT=1 export GO111MODULE=on -export GOPATH=${HOME}/.local +export GOPATH=/opt/gitlab/.local export HISTFILESIZE=0 export HISTSIZE=0 export LANG=C.UTF-8 @@ -98,7 +98,6 @@ function prepare_project() { ${SETUP_CMD[@]} PREPARE="--no-prepare" fi - gem install --local -f --silent "$LM_HOME/pkg/*.gem" license_management ignored_groups add development license_management ignored_groups add develop license_management ignored_groups add test diff --git a/spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile b/spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile new file mode 100644 index 0000000..d26dbc9 --- /dev/null +++ b/spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile @@ -0,0 +1,5 @@ +source 'https://rubygems.org' + +ruby '2.4.9' + +gem 'saml-kit' diff --git a/spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile.lock b/spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile.lock new file mode 100644 index 0000000..ccb0aef --- /dev/null +++ b/spec/fixtures/ruby/bundler/ruby-2.4.9/Gemfile.lock @@ -0,0 +1,47 @@ +GEM + remote: https://rubygems.org/ + specs: + activemodel (5.2.4.3) + activesupport (= 5.2.4.3) + activesupport (5.2.4.3) + concurrent-ruby (~> 1.0, >= 1.0.2) + i18n (>= 0.7, < 2) + minitest (~> 5.1) + tzinfo (~> 1.1) + builder (3.2.4) + concurrent-ruby (1.1.6) + i18n (1.8.3) + concurrent-ruby (~> 1.0) + mini_portile2 (2.4.0) + minitest (5.14.1) + net-hippie (0.3.2) + nokogiri (1.10.10) + mini_portile2 (~> 2.4.0) + saml-kit (1.1.0) + activemodel (>= 4.2.0) + net-hippie (~> 0.1) + xml-kit (>= 0.3.0, < 1.0.0) + thread_safe (0.3.6) + tilt (2.0.10) + tzinfo (1.2.7) + thread_safe (~> 0.1) + xml-kit (0.4.0) + activemodel (>= 4.2.0) + builder (~> 3.2) + nokogiri (~> 1.10) + tilt (>= 1.4.1) + xmldsig (~> 0.6) + xmldsig (0.6.6) + nokogiri (>= 1.6.8, < 2.0.0) + +PLATFORMS + ruby + +DEPENDENCIES + saml-kit + +RUBY VERSION + ruby 2.4.9p362 + +BUNDLED WITH + 2.1.4 diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb index c306997..81f9164 100644 --- a/spec/spec_helper.rb +++ b/spec/spec_helper.rb @@ -5,6 +5,8 @@ require 'json' require 'json-schema' require 'securerandom' require 'support/fixture_file_helper' +require 'support/report' +require 'support/project_helper' require 'support/integration_test_helper' require 'support/matchers' require 'support/proxy_helper' diff --git a/spec/support/integration_test_helper.rb b/spec/support/integration_test_helper.rb index 7dc64fe..244f363 100644 --- a/spec/support/integration_test_helper.rb +++ b/spec/support/integration_test_helper.rb @@ -1,95 +1,4 @@ module IntegrationTestHelper - class Report - attr_reader :report - - def initialize(raw) - @report = JSON.parse(raw, symbolize_names: true) - end - - def [](key) - report[key] - end - - def dependency_names - report[:dependencies].map { |x| x[:name] } - end - - def licenses_for(name) - (find(name) || {}).fetch(:licenses, []) - end - - def find(name) - report[:dependencies].find do |dependency| - dependency[:name] == name - end - end - - def nil? - report.nil? - end - - def to_hash - to_h - end - - def to_h - report - end - end - - class IntegrationTestRunner - attr_reader :project_path - - def initialize(project_path = Pathname.pwd.join('tmp').join(SecureRandom.uuid)) - FileUtils.mkdir_p(project_path) - @project_path = Pathname(project_path) - end - - def add_file(name, content = nil) - full_path = project_path.join(name) - FileUtils.mkdir_p(full_path.dirname) - IO.write(full_path, block_given? ? yield : content) - end - - def mount(dir:) - FileUtils.cp_r("#{dir}/.", project_path) - end - - def clone(repo, branch: 'master') - if branch.match?(/\b[0-9a-f]{5,40}\b/) - execute({}, 'git', 'clone', '--quiet', repo, project_path.to_s) - Dir.chdir project_path do - execute({}, 'git', 'checkout', branch) - end - else - execute({}, 'git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', branch, repo, project_path.to_s) - end - end - - def scan(env: {}) - execute(env, 'mkdir -p pkg/ && gem build --silent -o pkg/license-management-test.gem ./*.gemspec') - Dir.chdir project_path do - merged_env = { 'CI_PROJECT_DIR' => project_path.to_s }.merge(env) - return {} unless execute(merged_env, "#{License::Management.root.join('run.sh')} analyze .") - - report_path = project_path.join('gl-license-scanning-report.json') - return {} unless report_path.exist? - - Report.new(report_path.read) - end - end - - def execute(env = {}, *args) - Bundler.with_unbundled_env do - system(env, *args, exception: true) - end - end - - def cleanup - FileUtils.rm_rf(project_path) if project_path.exist? - end - end - def private_npm_host @private_npm_host ||= ENV.fetch('PRIVATE_NPM_HOST').tap do |host| add_host(host, ENV.fetch('PRIVATE_NPM_IP')) @@ -109,11 +18,10 @@ module IntegrationTestHelper end def runner(*args) - @runner ||= IntegrationTestRunner.new(*args) + @runner ||= ProjectHelper.new(*args) end def add_host(name, ip) - return unless ENV['LM_HOME'] return if system("grep #{name} /etc/hosts") system("echo '#{ip} #{name}' >> /etc/hosts") diff --git a/spec/support/project_helper.rb b/spec/support/project_helper.rb new file mode 100644 index 0000000..4ae490e --- /dev/null +++ b/spec/support/project_helper.rb @@ -0,0 +1,56 @@ +class ProjectHelper + attr_reader :project_path + + def initialize(project_path = License::Management.root.join('tmp').join(SecureRandom.uuid)) + FileUtils.mkdir_p(project_path) + @project_path = Pathname(project_path) + end + + def add_file(name, content = nil) + full_path = project_path.join(name) + FileUtils.mkdir_p(full_path.dirname) + IO.write(full_path, block_given? ? yield : content) + end + + def mount(dir:) + FileUtils.cp_r("#{dir}/.", project_path) + end + + def chdir + Dir.chdir project_path do + yield + end + end + + def clone(repo, branch: 'master') + if branch.match?(/\b[0-9a-f]{5,40}\b/) + execute({}, 'git', 'clone', '--quiet', repo, project_path.to_s) + chdir do + execute({}, 'git', 'checkout', branch) + end + else + execute({}, 'git', 'clone', '--quiet', '--depth=1', '--single-branch', '--branch', branch, repo, project_path.to_s) + end + end + + def scan(env: {}) + chdir do + return {} unless execute({ 'CI_PROJECT_DIR' => project_path.to_s }.merge(env), "#{License::Management.root.join('run.sh')} analyze .") + + report_path = project_path.join('gl-license-scanning-report.json') + return {} unless report_path.exist? + + Report.new(report_path.read) + end + end + + def execute(env = {}, *args) + Bundler.with_unbundled_env do + system(env, *args, exception: true) + end + end + + def cleanup + FileUtils.rm_rf(project_path) if project_path.exist? + end +end diff --git a/spec/support/report.rb b/spec/support/report.rb new file mode 100644 index 0000000..d6546ae --- /dev/null +++ b/spec/support/report.rb @@ -0,0 +1,37 @@ +class Report + attr_reader :report + + def initialize(raw) + @report = JSON.parse(raw, symbolize_names: true) + end + + def [](key) + report[key] + end + + def dependency_names + report[:dependencies].map { |x| x[:name] } + end + + def licenses_for(name) + (find(name) || {}).fetch(:licenses, []) + end + + def find(name) + report[:dependencies].find do |dependency| + dependency[:name] == name + end + end + + def nil? + report.nil? + end + + def to_hash + to_h + end + + def to_h + report + end +end diff --git a/spec/unit/license_finder/bundler_spec.rb b/spec/unit/license_finder/bundler_spec.rb new file mode 100644 index 0000000..5458892 --- /dev/null +++ b/spec/unit/license_finder/bundler_spec.rb @@ -0,0 +1,48 @@ +require 'spec_helper' + +RSpec.describe LicenseFinder::Bundler do + describe "#current_packages" do + subject do + project.chdir do + bundler.prepare + bundler.current_packages + end + end + + let(:bundler) { described_class.new(options) } + let(:options) { { ignored_groups: [], project_path: project.project_path } } + let(:project) { ProjectHelper.new } + + before do + project.mount(dir: project_fixture) + end + + after do + project.cleanup + end + + context "when scanning a v2.1 bundler project" do + let(:project_fixture) { fixture_file('ruby/bundler-v2.1') } + + specify { expect(subject.map(&:name)).to match_array(%w[bundler net-hippie]) } + end + + context "when scanning a v1.17 bundler project" do + let(:project_fixture) { fixture_file('ruby/bundler-v1.17') } + + specify { expect(subject.map(&:name).sort).to match_array(%w[activemodel activesupport builder bundler concurrent-ruby i18n mini_portile2 minitest net-hippie nokogiri saml-kit thread_safe tilt tzinfo xml-kit xmldsig zeitwerk]) } + end + + context "when scanning a project with a .ruby-version:2.4.9" do + let(:project_fixture) { fixture_file('ruby/bundler-ruby-2.4.9-no-lockfile') } + + specify { expect(subject.map(&:name)).to include("saml-kit") } + end + + context "when scanning a project with a Gemfile that specifies 2.4.9" do + let(:project_fixture) { fixture_file('ruby/bundler/ruby-2.4.9') } + + specify { expect(subject.map(&:name)).to include("saml-kit") } + end + end +end -- cgit v1.2.3