Convert python qa jobs to integration tests

18 files changed, 27 insertions, 139 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 302ab2c..026b6ac 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -10,7 +10,6 @@ include:
   - template: Container-Scanning.gitlab-ci.yml
   - template: Jobs/Code-Quality.gitlab-ci.yml
   - local: .gitlab/test.yml
-  - local: .gitlab/qa.yml
   - local: .gitlab/release.yml
 
 services:
diff --git a/.gitlab/qa.yml b/.gitlab/qa.yml
deleted file mode 100644
index 59c639c..0000000
--- a/.gitlab/qa.yml
+++ /dev/null
@@ -1,54 +0,0 @@
-.QA:
-  image: docker:stable
-  stage: test
-  variables:
-    LM_PYTHON_VERSION: 3
-    LM_REPORT_VERSION: 1
-  script:
-    - docker info
-    - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
-    - docker pull $TMP_IMAGE
-    - mkdir results
-    - IMAGE_NAME=$TMP_IMAGE ./bin/test
-  artifacts:
-    paths:
-      - results/
-    when: always
-
-QA:python-pip:
-  extends: .QA
-  variables:
-    LM_PYTHON_VERSION: 2
-    QA_PROJECT: python-pip
-    QA_REF: 04dce91b
-
-QA:python-pip-v1-1:
-  extends: QA:python-pip
-  variables:
-    LM_REPORT_VERSION: '1.1'
-    QA_RESULTS: python-pip-v1.1
-
-QA:python-pip-v2:
-  extends: QA:python-pip
-  variables:
-    LM_REPORT_VERSION: '2.0'
-    QA_RESULTS: python-pip-v2
-
-QA:python3-pip:
-  extends: .QA
-  variables:
-    QA_RESULTS: python3-pip-v1
-    QA_PROJECT: python-pip
-    QA_REF: 48e250a1
-
-QA:python3-pip-v1-1:
-  extends: QA:python3-pip
-  variables:
-    LM_REPORT_VERSION: '1.1'
-    QA_RESULTS: python3-pip-v1.1
-
-QA:python3-pip-v2:
-  extends: QA:python3-pip
-  variables:
-    LM_REPORT_VERSION: 2
-    QA_RESULTS: python3-pip-v2
diff --git a/README.md b/README.md
index b976fb7..8ae4865 100644
--- a/README.md
+++ b/README.md
@@ -19,7 +19,7 @@ only, but this may change in the future.
       registry.gitlab.com/gitlab-org/security-products/license-management:latest analyze /code
     ```
 
-1. The results will be stored in the `gl-license-management-report.json` file in the application directory. `gl-license-management-report.html` is also available with a human readable report.
+1. The results will be stored in the `gl-license-management-report.json` file in the application directory.
 
 ## Development
 
@@ -38,24 +38,12 @@ You can then run License Management on some target directory:
 docker run --rm --volume "/path/to/my/project":/code license-management analyze /code
 ```
 
-You can run the QA tests using the following command:
+You can run the tests using the following command:
 
 ```sh
-./bin/test-all
+./bin/test
 ```
 
-Or you can run a specific QA test using the following command:
-
-```sh
-QA_PROJECT=project_name QA_REF=git_ref ./bin/test
-```
-
-where:
-
-- `project_name` is the project slug among the
-[test projects for security products](https://gitlab.com/gitlab-org/security-products/tests/) (e.g. `java-maven`)
-- `git_ref` is the Git ref to checkout and scan against (e.g. `master`)
-
 ### Updating the SPDX index
 
 We will need to periodically update the SPDX index. This can be achieved with
@@ -149,7 +137,7 @@ unless a new package manager has been added and additional tooling needs to be i
 1. Bump the license management version in [CHANGELOG.md](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/CHANGELOG.md) and in [version.rb](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb)
 1. Update the `license_finder` version constraint in the [gemspec](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/39afdc40296871d9f7c1732ff3f7e1e3b7b28bbe/license-management.gemspec#L30).
 1. Run `bundle update license_finder`
-1. Test the changes locally using the `bin/test-all` script.
+1. Test the changes locally using the `bin/test` script.
 1. Submit a merge request.
 
 ### Upgrading the Docker image
@@ -159,7 +147,7 @@ unless a new package manager has been added and additional tooling needs to be i
 1. If an update is available, create a branch
 1. Bump the license management version in [CHANGELOG.md](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/CHANGELOG.md) and in [version.rb](https://gitlab.com/gitlab-org/security-products/license-management/-/blob/master/lib/license/management/version.rb)
 1. Edit the version specified in the `DockerFile` to match the desired `license_finder` Docker image tag.
-1. Test the changes locally using the `bin/test-all` script.
+1. Test the changes locally using the `bin/test` script.
 1. Submit a merge request.
 
 # Contributing
diff --git a/bin/test-local b/bin/docker-test
index 75bfc39..75bfc39 100755
--- a/bin/test-local
+++ b/bin/docker-test
diff --git a/bin/test b/bin/test
index 8c08e55..70f8581 100755
--- a/bin/test
+++ b/bin/test
@@ -4,15 +4,4 @@ set -e
 
 cd "$(dirname "$0")/.."
 
-export LM_PYTHON_VERSION=${LM_PYTHON_VERSION:-2}
-export LM_REPORT_VERSION=${LM_REPORT_VERSION:-1}
-export QA_RESULTS=${QA_RESULTS:-$QA_PROJECT-v$LM_REPORT_VERSION}
-export RESULTS_DIR=${RESULTS_DIR:-$(pwd)/results}
-export QA_REF=${QA_REF:-master}
-
-docker run \
-  --rm \
-  --env LM_PYTHON_VERSION \
-  --env LM_REPORT_VERSION \
-  --volume "$RESULTS_DIR":/results \
-  "$IMAGE_NAME" test "$QA_PROJECT" "$QA_RESULTS" "$QA_REF"
+bundle exec rspec
diff --git a/bin/test-all b/bin/test-all
deleted file mode 100755
index be92121..0000000
--- a/bin/test-all
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/bin/bash
-
-set -e
-
-cd "$(dirname "$0")/.."
-
-RESULTS_DIR=$(pwd)/tmp
-export RESULTS_DIR
-IMAGE_NAME=$(basename "$PWD"):latest
-export IMAGE_NAME
-
-./bin/docker-build
-
-REPORT_VERSIONS=(2 1.1 1)
-for version in "${REPORT_VERSIONS[@]}"
-do
-  export LM_REPORT_VERSION=$version
-  echo "Report Version $LM_REPORT_VERSION"
-  QA_PROJECT=go-modules ./bin/test
-  QA_PROJECT=php-composer ./bin/test
-  QA_PROJECT=java-maven QA_REF=831c7a04 ./bin/test
-  QA_PROJECT=python-pip QA_REF=04dce91b LM_PYTHON_VERSION=2 ./bin/test
-  QA_PROJECT=python-pip QA_RESULTS="python3-pip-v$version" QA_REF=48e250a1 LM_PYTHON_VERSION=3 ./bin/test
-  QA_PROJECT=ruby-bundler ./bin/test
-  QA_PROJECT=js-yarn ./bin/test
-  QA_PROJECT=js-npm ./bin/test
-  QA_PROJECT=csharp-nuget-dotnetcore ./bin/test
-  QA_PROJECT=python-pipenv QA_REF=pip-file-lock ./bin/test
-  QA_PROJECT=java-gradle ./bin/test
-done
diff --git a/run.sh b/run.sh
index 4e85ae7..09b268a 100755
--- a/run.sh
+++ b/run.sh
@@ -68,27 +68,4 @@ function scan_project() {
   license_management report "${options[@]}"
 }
 
-function test_project() {
-  project=$1
-  results=$2
-  ref=$3
-  url="https://gitlab.com/gitlab-org/security-products/tests/$project.git"
-  echo "$project" "$results" "$ref" "$url"
-
-  echo "Cloning the test project $project from $url"
-  git clone "$url" "/code/$project"
-
-  echo "Running license management on the $project with ref $ref"
-  cd "/code/$project"
-  git checkout "$ref"
-  scan_project "/code/$project"
-
-  mkdir -p /results/
-  cp "/code/$project/gl-license-management-report.json" "/results/$project-gl-license-management-report.json"
-
-  diff -u "/test/results/$results.json" "/code/$project/gl-license-management-report.json"
-  echo "OK"
-}
-
-echo "$@"
-scan_project "$1"
+scan_project "$2"
diff --git a/spec/fixtures/expected/python-pip-v1.json b/spec/fixtures/expected/python/2/pip/v1.0.json
index 5adaedd..5adaedd 100644
--- a/spec/fixtures/expected/python-pip-v1.json
+++ b/spec/fixtures/expected/python/2/pip/v1.0.json
diff --git a/spec/fixtures/expected/python-pip-v1.1.json b/spec/fixtures/expected/python/2/pip/v1.1.json
index 30e4cc1..30e4cc1 100644
--- a/spec/fixtures/expected/python-pip-v1.1.json
+++ b/spec/fixtures/expected/python/2/pip/v1.1.json
diff --git a/spec/fixtures/expected/python-pip-v2.json b/spec/fixtures/expected/python/2/pip/v2.0.json
index 1e675ff..1e675ff 100644
--- a/spec/fixtures/expected/python-pip-v2.json
+++ b/spec/fixtures/expected/python/2/pip/v2.0.json
diff --git a/spec/fixtures/expected/python3-pip-v1.json b/spec/fixtures/expected/python/3/pip/v1.0.json
index 6cf3676..6cf3676 100644
--- a/spec/fixtures/expected/python3-pip-v1.json
+++ b/spec/fixtures/expected/python/3/pip/v1.0.json
diff --git a/spec/fixtures/expected/python3-pip-v1.1.json b/spec/fixtures/expected/python/3/pip/v1.1.json
index 580d4cb..580d4cb 100644
--- a/spec/fixtures/expected/python3-pip-v1.1.json
+++ b/spec/fixtures/expected/python/3/pip/v1.1.json
diff --git a/spec/fixtures/expected/python3-pip-v2.json b/spec/fixtures/expected/python/3/pip/v2.0.json
index 5d885ee..5d885ee 100644
--- a/spec/fixtures/expected/python3-pip-v2.json
+++ b/spec/fixtures/expected/python/3/pip/v2.0.json
diff --git a/spec/fixtures/expected/python-pipenv-v1.json b/spec/fixtures/expected/python/pipenv/v1.0.json
index 6c0ae63..6c0ae63 100644
--- a/spec/fixtures/expected/python-pipenv-v1.json
+++ b/spec/fixtures/expected/python/pipenv/v1.0.json
diff --git a/spec/fixtures/expected/python-pipenv-v1.1.json b/spec/fixtures/expected/python/pipenv/v1.1.json
index 0528f88..0528f88 100644
--- a/spec/fixtures/expected/python-pipenv-v1.1.json
+++ b/spec/fixtures/expected/python/pipenv/v1.1.json
diff --git a/spec/fixtures/expected/python-pipenv-v2.json b/spec/fixtures/expected/python/pipenv/v2.0.json
index bdbeb14..bdbeb14 100644
--- a/spec/fixtures/expected/python-pipenv-v2.json
+++ b/spec/fixtures/expected/python/pipenv/v2.0.json
diff --git a/spec/integration/python/pip_spec.rb b/spec/integration/python/pip_spec.rb
index b092189..0853a71 100644
--- a/spec/integration/python/pip_spec.rb
+++ b/spec/integration/python/pip_spec.rb
@@ -46,4 +46,23 @@ RSpec.describe "pip" do
       expect(report[:dependencies]).not_to be_empty
     end
   end
+
+  ['2', '3'].each do |python_version|
+    ['1.0', '1.1', '2.0'].each do |report_version|
+      context "when generating a `#{report_version}` report using Python `#{python_version}`" do
+        let(:url) { "https://gitlab.com/gitlab-org/security-products/tests/#{language}-#{package_manager}.git" }
+        let(:language) { 'python' }
+        let(:package_manager) { 'pip' }
+        let(:environment) { { 'LM_REPORT_VERSION' => report_version, 'LM_PYTHON_VERSION' => python_version } }
+
+        it 'matches the expected report' do
+          runner.clone(url, branch: '48e250a1')
+          report = runner.scan(env: environment)
+          #expect(report).to match_schema(version: report_version)
+          expected = JSON.parse(fixture_file_content("expected/#{language}/#{python_version}/#{package_manager}/v#{report_version}.json"))
+          expect(report).to eql(expected)
+        end
+      end
+    end
+  end
 end
diff --git a/spec/support/integration_test_helper.rb b/spec/support/integration_test_helper.rb
index 23f7970..b937f1b 100644
--- a/spec/support/integration_test_helper.rb
+++ b/spec/support/integration_test_helper.rb
@@ -18,7 +18,7 @@ module IntegrationTestHelper
     end
 
     def scan(env: {})
-      return {} unless execute(env, './bin/test-local', project_path)
+      return {} unless execute(env, './bin/docker-test', project_path)
 
       report_path = "#{project_path}/gl-license-management-report.json"
       return {} unless File.exist?(report_path)