Parallelize QA tests

Test projects are now given as a parameter to the test command

3 files changed, 57 insertions, 28 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1190750..2c63d4b 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -60,14 +60,31 @@ container_scanning:
     reports:
       container_scanning: gl-container-scanning-report.json
 
-QA:
+.QA:
   image: docker:stable
   stage: test
   script:
     - docker info
     - docker login -u gitlab-ci-token -p $CI_JOB_TOKEN $CI_REGISTRY
     - docker pull $TMP_IMAGE
-    - docker run $TMP_IMAGE test
+    - mkdir results
+    - docker run --volume `pwd`/results:/results $TMP_IMAGE test $QA_PROJECT $QA_REF
+  artifacts:
+    paths:
+      - results/
+    when: always
+
+QA:java-maven:
+  extends: .QA
+  variables:
+    QA_PROJECT: java-maven
+    QA_REF: ccec8318
+
+QA:ruby-bundler:
+  extends: .QA
+  variables:
+    QA_PROJECT: ruby-bundler
+    QA_REF: 6b858821
 
 .docker_tag:
   image: docker:stable
diff --git a/run.sh b/run.sh
index d82d133..bba4771 100755
--- a/run.sh
+++ b/run.sh
@@ -7,7 +7,7 @@ $(basename "$0") [-h]
 
 $(basename "$0") analyze PROJECT_PATH
 
-$(basename "$0") test
+$(basename "$0") test PROJECT_NAME PROJECT_REF
 
 where:
   -h  show this help text
@@ -60,7 +60,7 @@ if [ "$COMMAND" = "analyze" -a $# -ne 1 ] ; then
   exit 1
 fi
 
-if [ "$COMMAND" = "test" -a $# -ne 0 ] ; then
+if [ "$COMMAND" = "test" -a $# -ne 2 ] ; then
   echo "$usage"
   exit 1
 fi
@@ -69,7 +69,7 @@ fi
 case "$COMMAND" in
   test)
     # Run integration tests.
-    exec /test/test.sh
+    exec /test/test.sh $1 $2
     ;;
 
   analyze)
diff --git a/test/test.sh b/test/test.sh
index 53c1805..bbf66f7 100755
--- a/test/test.sh
+++ b/test/test.sh
@@ -2,26 +2,38 @@
 
 set -e
 
-for project in ruby-bundler-rails java-maven; do
-  # Clone the test repository.
-  echo "Cloning the test repository https://gitlab.com/gitlab-org/security-products/tests/$project.git"
-  git clone "https://gitlab.com/gitlab-org/security-products/tests/$project.git" "/code/$project"
-
-  # Run license management on it.
-  echo "Running license management on the $project"
-  cd  "/code/$project"
-  /run.sh analyze .
-
-  # Compare results with expected results.
-  set +e
-  diff "/code/$project/gl-license-management-report.json" "/test/results/$project.json" > /diff.txt
-  set -e
-
-  if [ -s /diff.txt ] ; then
-    echo "Unexpected result. Here is the diff between actual results and those expected :"
-    cat /diff.txt
-    exit 1
-  fi
-done
-
-echo "All tests are OK."
+project=$1
+ref=$2
+url="https://gitlab.com/gitlab-org/security-products/tests/$project.git"
+
+# Clone the test repository.
+echo "Cloning the test project $project from $url"
+git clone "$url" "/code/$project"
+
+# Run license management on it.
+echo "Running license management on the $project with ref $ref"
+cd  "/code/$project"
+git checkout "$ref" 2> /dev/null
+/run.sh analyze .
+
+mkdir -p /results/
+# In order to upload the artifact always
+cp "/code/$project/gl-license-management-report.json" "/results/$project-gl-license-management-report.json"
+
+# Compare results with expected results.
+set +e
+
+diff "/code/$project/gl-license-management-report.json" "/test/results/$project.json" > /diff.txt
+error=$?
+if [[ $error -eq 0 ]]; then
+   echo "The report matches the fixture."
+   exit 0
+elif [[ $error -eq 1 ]]; then
+   echo "Unexpected result. Here is the diff between actual results and those expected :"
+   cat /diff.txt
+   mv /diff.txt "/results/$project-diff.txt"
+else
+  echo "Could not compare the fixture to the generated report"
+fi
+
+exit 1