Merge pull request #28 from CybercentreCanada/update/push-to-azure

Update/push to azure [dev]

README.md

@@ -39,6 +39,7 @@ Once MetaDefender Core has been installed and activated with your license, the f
 * **base_url**: The URL(s) of the MetaDefender deployment(s)
 	* If you have a **single** MetaDefender Core deployment, set the service variable to **str** type and enter the URL of your MetaDefender Core deployment
 	* If you have **multiple** MetaDefender Core deployments, set the service variable to **list** type and enter the URLs of your MetaDefender Core deployments separated by a comma
+* **verify_certificate**: Setting to False will ignore verifying the SSL certificate
 * **md_version**: Version of MetaDefender you're connecting to (3 or 4)
 * **md_timeout**: Maximum amount of time to wait while connecting to the MetaDefender server
 * **max_md_scan_time**: Maximum amount of time to wait for scan results before the MetaDefender server is put on a brief timeout (only applicable when multiple MetaDefender deployments are used)

metadefender.py

@@ -6,12 +6,15 @@
 import json
 from requests import Session, Response, ConnectionError, exceptions, codes
 
+from assemblyline.common import forge
 from assemblyline.common.exceptions import RecoverableError
 from assemblyline.common.isotime import iso_to_local, iso_to_epoch, epoch_to_local, now, now_as_local
 from assemblyline_v4_service.common.api import ServiceAPIError
 from assemblyline_v4_service.common.base import ServiceBase
 from assemblyline_v4_service.common.request import ServiceRequest
-from assemblyline_v4_service.common.result import Result, ResultSection, Classification, BODY_FORMAT
+from assemblyline_v4_service.common.result import Result, ResultSection, BODY_FORMAT
+
+Classification = forge.get_classification()
 
 
 class AvHitSection(ResultSection):
@@ -73,6 +76,7 @@ def __init__(self, config: Optional[Dict[str, Any]] = None) -> None:
         self.kw_score_revision_map: Optional[Dict[str, int]] = None
         self.sig_score_revision_map: Optional[Dict[str, Any]] = None
         self.safelist_match: List[str] = []
+        self.verify = self.config.get("verify_certificate", True)
         api_key = self.config.get("api_key")
         if api_key:
             self.headers = {"apikey": api_key}
@@ -173,7 +177,7 @@ def _get_version_map(self, node: str) -> None:
 
         try:
             self.log.debug(f"_get_version_map: GET {url}")
-            r = self.session.get(url=url, timeout=self.timeout)
+            r = self.session.get(url=url, timeout=self.timeout, verify=self.verify)
             engines = r.json()
 
             for engine in engines:
@@ -268,7 +272,7 @@ def get_scan_results_by_data_id(self, data_id: str) -> Response:
 
         try:
             self.log.debug(f"get_scan_results_by_data_id: GET {url}")
-            return self.session.get(url=url, headers=self.headers, timeout=self.timeout)
+            return self.session.get(url=url, headers=self.headers, timeout=self.timeout, verify=self.verify)
         except exceptions.Timeout:
             self.new_node(force=True, reset_queue=True)
             raise Exception(f"Node ({self.current_node}) timed out after {self.timeout}s "
@@ -332,7 +336,7 @@ def scan_file(self, filename: str) -> Dict[str, Any]:
 
         try:
             self.log.debug(f"scan_file: POST {url}")
-            r = self.session.post(url=url, data=data, headers=self.headers, timeout=self.timeout)
+            r = self.session.post(url=url, data=data, headers=self.headers, timeout=self.timeout, verify=self.verify)
         except exceptions.Timeout:
             self.new_node(force=True, reset_queue=True)
             raise Exception(f"Node ({self.current_node}) timed out after {self.timeout}s "

pipelines/azure-build.yaml

@@ -1,5 +1,25 @@
 name: build
 
+variables:
+  - group: unittest-samples
+  - name: self_location
+    value: "self_location"
+  - name: full_self_location
+    value: "$(Agent.BuildDirectory)/$(self_location)"
+  - name: samples_location
+    value: "samples_location"
+  - name: full_samples_location
+    value: "$(Agent.BuildDirectory)/$(samples_location)"
+
+resources:
+  repositories:
+  - repository: unittest-samples
+    type: github
+    name: $(unittest_samples_repository)
+    ref: main
+    endpoint: github-repo-sa
+    trigger: none
+
 trigger:
   tags:
     include: ["v*"]
@@ -21,10 +41,34 @@ stages:
             inputs:
               command: login
               containerRegistry: dockerhub
+          - checkout: self
+            fetchDepth: 1
+            path: $(self_location)
+          - checkout: unittest-samples
+            fetchDepth: 1
+            path: $(samples_location)
+          - script: |
+              export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
+              if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
+              docker build --build-arg version=$TAG --build-arg branch=$BUILD_TYPE -t cccs/${BUILD_REPOSITORY_NAME##*/}:$TAG -t cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE -f ./Dockerfile .
+            workingDirectory: $(full_self_location)
+            displayName: Build containers
           - script: |
-              set -xv  # Echo commands before they are run
+              [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit
               export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
               if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
-              docker build --build-arg version=$TAG --build-arg branch=$BUILD_TYPE -t cccs/assemblyline-service-metadefender:$TAG -t cccs/assemblyline-service-metadefender:$BUILD_TYPE .
-              docker push cccs/assemblyline-service-metadefender --all-tags
-            displayName: Deploy to Docker Hub
+              [ -f "$(pwd)/tests/requirements.txt" ] && docker run -e FULL_SELF_LOCATION=/opt/al_service -e FULL_SAMPLES_LOCATION=/opt/samples -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla -v $(pwd)/tests/:/opt/al_service/tests/ -v ${FULL_SAMPLES_LOCATION}:/opt/samples cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE bash -c 'pip install -U -r tests/requirements.txt; pytest -p no:cacheprovider -vv' && exit
+              docker run -e FULL_SELF_LOCATION=/opt/al_service -e FULL_SAMPLES_LOCATION=/opt/samples -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla -v $(pwd)/tests/:/opt/al_service/tests/ -v ${FULL_SAMPLES_LOCATION}:/opt/samples cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE bash -c 'pytest -p no:cacheprovider -vv'
+            workingDirectory: $(full_self_location)
+            displayName: Test containers
+          - script: |
+              export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
+              if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
+
+              for IMAGE in "cccs/" "uchimera.azurecr.io/cccs/"
+              do
+                docker tag ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$TAG
+                docker tag ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE
+                docker push ${IMAGE}${BUILD_REPOSITORY_NAME##*/} --all-tags
+              done
+            displayName: Deploy to container repositories

service_manifest.yml

@@ -20,6 +20,7 @@ licence_count: 0
 config:
   api_key: ""
   base_url: 'http://localhost:8008/'
+  verify_certificate: true
   md_version: 4
   md_timeout: 40
   min_node_time: 60