From 32df855a47406fec8684e17225d34bc0834f7940 Mon Sep 17 00:00:00 2001 From: ElJeffe Date: Thu, 1 Sep 2022 17:11:50 -0400 Subject: [PATCH 1/3] verify_certificate Helpful if you're using self-signed certs or internal / external hostname issues. Default is True. --- README.md | 1 + metadefender.py | 7 ++++--- service_manifest.yml | 1 + 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index c4da6ae..0594807 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,7 @@ Once MetaDefender Core has been installed and activated with your license, the f * **base_url**: The URL(s) of the MetaDefender deployment(s) * If you have a **single** MetaDefender Core deployment, set the service variable to **str** type and enter the URL of your MetaDefender Core deployment * If you have **multiple** MetaDefender Core deployments, set the service variable to **list** type and enter the URLs of your MetaDefender Core deployments separated by a comma +* **verify_certificate**: Setting to False will ignore verifying the SSL certificate * **md_version**: Version of MetaDefender you're connecting to (3 or 4) * **md_timeout**: Maximum amount of time to wait while connecting to the MetaDefender server * **max_md_scan_time**: Maximum amount of time to wait for scan results before the MetaDefender server is put on a brief timeout (only applicable when multiple MetaDefender deployments are used) diff --git a/metadefender.py b/metadefender.py index 77ed5f0..aa4a590 100644 --- a/metadefender.py +++ b/metadefender.py @@ -73,6 +73,7 @@ def __init__(self, config: Optional[Dict[str, Any]] = None) -> None: self.kw_score_revision_map: Optional[Dict[str, int]] = None self.sig_score_revision_map: Optional[Dict[str, Any]] = None self.safelist_match: List[str] = [] + self.verify = self.config.get("verify_certificate", True) api_key = self.config.get("api_key") if api_key: self.headers = {"apikey": api_key} @@ -173,7 +174,7 @@ def _get_version_map(self, node: str) -> None: try: self.log.debug(f"_get_version_map: GET {url}") - r = self.session.get(url=url, timeout=self.timeout) + r = self.session.get(url=url, timeout=self.timeout, verify=self.verify) engines = r.json() for engine in engines: @@ -268,7 +269,7 @@ def get_scan_results_by_data_id(self, data_id: str) -> Response: try: self.log.debug(f"get_scan_results_by_data_id: GET {url}") - return self.session.get(url=url, headers=self.headers, timeout=self.timeout) + return self.session.get(url=url, headers=self.headers, timeout=self.timeout, verify=self.verify) except exceptions.Timeout: self.new_node(force=True, reset_queue=True) raise Exception(f"Node ({self.current_node}) timed out after {self.timeout}s " @@ -332,7 +333,7 @@ def scan_file(self, filename: str) -> Dict[str, Any]: try: self.log.debug(f"scan_file: POST {url}") - r = self.session.post(url=url, data=data, headers=self.headers, timeout=self.timeout) + r = self.session.post(url=url, data=data, headers=self.headers, timeout=self.timeout, verify=self.verify) except exceptions.Timeout: self.new_node(force=True, reset_queue=True) raise Exception(f"Node ({self.current_node}) timed out after {self.timeout}s " diff --git a/service_manifest.yml b/service_manifest.yml index f0c119c..4929a26 100644 --- a/service_manifest.yml +++ b/service_manifest.yml @@ -20,6 +20,7 @@ licence_count: 0 config: api_key: "" base_url: 'http://localhost:8008/' + verify_certificate: true md_version: 4 md_timeout: 40 min_node_time: 60 From 284dfe4fa6f55764cd5976f0dad4ee29b038c653 Mon Sep 17 00:00:00 2001 From: Steve Garon Date: Mon, 17 Oct 2022 13:33:59 +0000 Subject: [PATCH 2/3] Fix classification object --- metadefender.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/metadefender.py b/metadefender.py index aa4a590..af0badd 100644 --- a/metadefender.py +++ b/metadefender.py @@ -6,12 +6,15 @@ import json from requests import Session, Response, ConnectionError, exceptions, codes +from assemblyline.common import forge from assemblyline.common.exceptions import RecoverableError from assemblyline.common.isotime import iso_to_local, iso_to_epoch, epoch_to_local, now, now_as_local from assemblyline_v4_service.common.api import ServiceAPIError from assemblyline_v4_service.common.base import ServiceBase from assemblyline_v4_service.common.request import ServiceRequest -from assemblyline_v4_service.common.result import Result, ResultSection, Classification, BODY_FORMAT +from assemblyline_v4_service.common.result import Result, ResultSection, BODY_FORMAT + +Classification = forge.get_classification() class AvHitSection(ResultSection): From 7bead4cb3783b1eab3167a7f2dbff6f39dc4ce6f Mon Sep 17 00:00:00 2001 From: cccs-kevin Date: Mon, 14 Nov 2022 18:50:32 +0000 Subject: [PATCH 3/3] Push to the ACR too --- pipelines/azure-build.yaml | 52 +++++++++++++++++++++++++++++++++++--- 1 file changed, 48 insertions(+), 4 deletions(-) diff --git a/pipelines/azure-build.yaml b/pipelines/azure-build.yaml index 7995d67..50e716c 100644 --- a/pipelines/azure-build.yaml +++ b/pipelines/azure-build.yaml @@ -1,5 +1,25 @@ name: build +variables: + - group: unittest-samples + - name: self_location + value: "self_location" + - name: full_self_location + value: "$(Agent.BuildDirectory)/$(self_location)" + - name: samples_location + value: "samples_location" + - name: full_samples_location + value: "$(Agent.BuildDirectory)/$(samples_location)" + +resources: + repositories: + - repository: unittest-samples + type: github + name: $(unittest_samples_repository) + ref: main + endpoint: github-repo-sa + trigger: none + trigger: tags: include: ["v*"] @@ -21,10 +41,34 @@ stages: inputs: command: login containerRegistry: dockerhub + - checkout: self + fetchDepth: 1 + path: $(self_location) + - checkout: unittest-samples + fetchDepth: 1 + path: $(samples_location) + - script: | + export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"} + if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi + docker build --build-arg version=$TAG --build-arg branch=$BUILD_TYPE -t cccs/${BUILD_REPOSITORY_NAME##*/}:$TAG -t cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE -f ./Dockerfile . + workingDirectory: $(full_self_location) + displayName: Build containers - script: | - set -xv # Echo commands before they are run + [ ! -d "$(pwd)/tests" ] && echo "No tests found" && exit export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"} if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi - docker build --build-arg version=$TAG --build-arg branch=$BUILD_TYPE -t cccs/assemblyline-service-metadefender:$TAG -t cccs/assemblyline-service-metadefender:$BUILD_TYPE . - docker push cccs/assemblyline-service-metadefender --all-tags - displayName: Deploy to Docker Hub + [ -f "$(pwd)/tests/requirements.txt" ] && docker run -e FULL_SELF_LOCATION=/opt/al_service -e FULL_SAMPLES_LOCATION=/opt/samples -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla -v $(pwd)/tests/:/opt/al_service/tests/ -v ${FULL_SAMPLES_LOCATION}:/opt/samples cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE bash -c 'pip install -U -r tests/requirements.txt; pytest -p no:cacheprovider -vv' && exit + docker run -e FULL_SELF_LOCATION=/opt/al_service -e FULL_SAMPLES_LOCATION=/opt/samples -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla -v $(pwd)/tests/:/opt/al_service/tests/ -v ${FULL_SAMPLES_LOCATION}:/opt/samples cccs/${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE bash -c 'pytest -p no:cacheprovider -vv' + workingDirectory: $(full_self_location) + displayName: Test containers + - script: | + export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"} + if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi + + for IMAGE in "cccs/" "uchimera.azurecr.io/cccs/" + do + docker tag ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$TAG + docker tag ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE ${IMAGE}${BUILD_REPOSITORY_NAME##*/}:$BUILD_TYPE + docker push ${IMAGE}${BUILD_REPOSITORY_NAME##*/} --all-tags + done + displayName: Deploy to container repositories