From a6333c707806895e29b86d49de5ee5d74bfb7a8b Mon Sep 17 00:00:00 2001 From: cccs-rs <62077998+cccs-rs@users.noreply.github.com> Date: Fri, 10 Jan 2025 17:29:57 +0000 Subject: [PATCH] Update project to follow template --- .cruft.json | 23 +++ .dockerignore | 14 +- .gitignore | 144 +++++++++++++++- .vscode/launch.json | 36 ++++ .vscode/settings.json | 32 ++++ CONTRIBUTING.md | 58 ++++++- Dockerfile | 26 ++- LICENCE.md | 11 -- LICENSE | 27 +++ Makefile | 26 +++ README.md | 156 ++++++++++++++++-- metadefender/__init__.py | 0 .../metadefender.py | 0 pyproject.toml | 2 + requirements.txt | 2 + service_manifest.yml | 14 +- tests/gentests.py | 30 ++++ tests/gentests.sh | 24 +++ tests/pytest.sh | 24 +++ {test => tests}/requirements.txt | 2 + ...b701b9ee9f99ef83b4cd07b695111d37eb95abcff8 | 0 {test => tests}/test_metadefender.py | 6 +- tests/test_metadefender_samples.py | 24 +++ 23 files changed, 631 insertions(+), 50 deletions(-) create mode 100644 .cruft.json create mode 100644 .vscode/launch.json create mode 100644 .vscode/settings.json delete mode 100644 LICENCE.md create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 metadefender/__init__.py rename metadefender.py => metadefender/metadefender.py (100%) create mode 100644 pyproject.toml create mode 100644 requirements.txt create mode 100755 tests/gentests.py create mode 100755 tests/gentests.sh create mode 100755 tests/pytest.sh rename {test => tests}/requirements.txt (54%) rename {test => tests}/samples/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8 (100%) rename {test => tests}/test_metadefender.py (99%) create mode 100644 tests/test_metadefender_samples.py diff --git a/.cruft.json b/.cruft.json new file mode 100644 index 0000000..32e8fc5 --- /dev/null +++ b/.cruft.json @@ -0,0 +1,23 @@ +{ + "template": "https://github.com/CybercentreCanada/assemblyline-service-template.git", + "commit": "349a8390e48c1607f6dc245add2627df07c7bc9b", + "checkout": null, + "context": { + "cookiecutter": { + "service_name": "metadefender", + "__svc_name": "metadefender", + "__repository": "assemblyline-service-metadefender", + "__pkg_name": "metadefender", + "class_name": "MetaDefender", + "short_description": "This Assemblyline service interfaces with the [MetaDefender Core](https://www.opswat.com/metadefender-core) multi-scanning AV engine.", + "short_description_fr": "Ce service Assemblyline s'interface avec le moteur AV à balayage multiple de [MetaDefender Core] (https://www.opswat.com/metadefender-core).", + "stage": "CORE", + "category": "Antivirus", + "org_name_full": "CybercentreCanada", + "org_name_short": "cccs", + "license": "mit", + "_template": "https://github.com/CybercentreCanada/assemblyline-service-template.git" + } + }, + "directory": null +} diff --git a/.dockerignore b/.dockerignore index a764cee..5f78d7b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,15 +1,25 @@ Dockerfile .idea .git +.gitignore +.vscode +.dockerignore pipelines venv +.venv env +.env test tests -exemples +examples docs +build +dist +**/__pycache__ +**/*.pyc + pip-log.txt pip-delete-this-directory.txt .tox @@ -18,5 +28,5 @@ pip-delete-this-directory.txt .cache nosetests.xml coverage.xml -*,cover +*.cover *.log diff --git a/.gitignore b/.gitignore index 9c07ba8..78f6696 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,7 @@ +# Created by https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode +# Edit at https://www.toptal.com/developers/gitignore?templates=python,vim,visualstudiocode + +### Python ### # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -6,11 +10,6 @@ __pycache__/ # C extensions *.so -# IDE files -.pydevproject -.python-version -.idea - # Distribution / packaging .Python build/ @@ -60,6 +59,22 @@ cover/ *.mo *.pot +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + # PyBuilder .pybuilder/ target/ @@ -71,6 +86,43 @@ target/ profile_default/ ipython_config.py +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + # Environments .env .venv @@ -80,5 +132,85 @@ ENV/ env.bak/ venv.bak/ +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + # Cython debug symbols -cython_debug/ \ No newline at end of file +cython_debug/ + +# PyCharm +# JetBrains specific template is maintained in a separate JetBrains.gitignore that can +# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore +# and can be added to the global gitignore or merged into this file. For a more nuclear +# option (not recommended) you can uncomment the following to ignore the entire idea folder. +#.idea/ + +### Python Patch ### +# Poetry local configuration file - https://python-poetry.org/docs/configuration/#local-configuration +poetry.toml + +# ruff +.ruff_cache/ + +# LSP config files +pyrightconfig.json + +### Vim ### +# Swap +[._]*.s[a-v][a-z] +!*.svg # comment out if you don't need vector files +[._]*.sw[a-p] +[._]s[a-rt-v][a-z] +[._]ss[a-gi-z] +[._]sw[a-p] + +# Session +Session.vim +Sessionx.vim + +# Temporary +.netrwhist +*~ +# Auto-generated tag files +tags +# Persistent undo +[._]*.un~ + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +!.vscode/*.code-snippets + +# Local History for Visual Studio Code +.history/ + +# Built Visual Studio Code Extensions +*.vsix + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# End of https://www.toptal.com/developers/gitignore/api/python,vim,visualstudiocode diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..4920214 --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,36 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "RunServiceOnce MetaDefender", + "type": "python", + "request": "launch", + "module": "assemblyline_v4_service.dev.run_service_once", + "cwd": "${workspaceFolder}", + "args": [ + "-d", + "metadefender.metadefender.MetaDefender", + "${file}" + ], + "justMyCode": false, + }, + { + "name": "[Service] MetaDefender - Privileged", + "type": "python", + "request": "launch", + "module": "assemblyline_v4_service.run_privileged_service", + "env": { + "SERVICE_MANIFEST_PATH": "service_manifest.yml", + "PRIVILEGED": "true", + "SERVICE_PATH": "metadefender.metadefender.MetaDefender", + "TASKING_DIR": "/tmp/MetaDefender" + }, + "console": "internalConsole", + "cwd": "${workspaceFolder}", + "justMyCode": false, + } + ] +} diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..54dc624 --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,32 @@ +{ + "editor.codeActionsOnSave": { + "source.organizeImports": "explicit" + }, + "editor.formatOnSave": true, + "editor.rulers": [ + 120 + ], + "editor.tabSize": 4, + "editor.wordWrap": "wordWrapColumn", + "editor.wordWrapColumn": 120, + "files.insertFinalNewline": true, + "files.trimFinalNewlines": true, + "files.trimTrailingWhitespace": true, + "isort.args": [ + "-l", + "120", + "--profile=black", + // "--src=${workspaceFolder}" + ], + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + }, + "black-formatter.args": [ + "--line-length=120" + ], + "flake8.args": [ + "--max-line-length=120", + //Added the ignore of E203 for now : https://github.com/PyCQA/pycodestyle/issues/373 + "--ignore=E203,W503" + ], +} diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6fb6aeb..7b78163 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,24 +2,68 @@ This guide covers the basics of how to contribute to the Assemblyline project. -Python code should follow the PEP8 guidelines defined here: [PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/). +Python code should follow the PEP8 guidelines defined here: +[PEP8 Guidelines](https://www.python.org/dev/peps/pep-0008/). ## Tell us want you want to build/fix -Before you start coding anything you should connect with the [Assemblyline community](https://groups.google.com/d/forum/cse-cst-assemblyline) to make sure no one else is working on the same thing and that whatever you are going to build still fits with the vision off the system. + +Before you start coding anything you should connect with the Assemblyline community via the +[Assemblyline Discord server](https://discord.gg/GUAy9wErNu) and/or the +[central Assemblyline GitHub project](https://github.com/CybercentreCanada/assemblyline/issues) to make sure no one +else is working on the same thing and that whatever you are going to build still fits with the vision of the system. ## Git workflow - Clone the repo to your own account - Checkout and pull the latest commits from the master branch - Make a branch -- Work in any way you like and make sure your changes actually work -- When you're satisfied with your changes, create a pull requests to the main assemblyline repo +- Work on your modifications and make sure your changes work as expected +- When you're satisfied with your changes, create a pull requests to the Assemblyline repo #### Transfer your service repo -If you've worked on a new service that you want to be included in the default service selection you'll have to transfer the repo into our control. + +If you've worked on a new service that you want to be included in the default service selection you'll have to transfer +the associated repo into our control. #### You are not allow to merge: -Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge code into our repositories. +Even if you try to merge in your pull request, you will be denied. Only a few people in our team are allowed to merge +code into our repositories. + +We check for new pull requests every day and will merge them in once they have been approved by someone in our team. + +# Guide de contribution d'Assemblyline + +Ce guide couvre les bases afin de contribuer au projet Assemblyline. + +Le code Python doit suivre les directives PEP8 définies ici: +[Directives PEP8](https://www.python.org/dev/peps/pep-0008/). + +## Dites-nous que vous voulez construire / réparer + +Avant de commencer à coder quoi que ce soit, vous devriez vous connecter à la communauté Assemblyline via le +[Serveur Discord Assemblyline](https://discord.gg/GUAy9wErNu) et/ou le +[projet GitHub central Assemblyline](https://github.com/CybercentreCanada/assemblyline/issues) pour vous assurer que +personne d'autre ne travaille sur la même chose et que tout ce que vous allez construire correspond toujours à la vision +du système. + +## Flux de travail avec Git + +- Clonez le référentiel sur votre propre compte +- Changez de branche pour la branche principale et la synchroniser avec le serveur de référence +- Faire une nouvelle branche +- Travaillez sur ce que vous souhaitez et assurez-vous que vos modifications fonctionnent comme prévu +- Lorsque vous êtes satisfait de vos modifications, créez une demande de fusion sur le référentiel d'Assemblyline + +#### Transférer votre référentiel de service + +Si vous avez travaillé sur un nouveau service que vous souhaitez inclure dans la sélection de service par défaut, vous +devrez transférer le référentiel associé sous notre contrôle. + +#### Vous n'êtes pas autorisé à compléter une fusion: + +Même si vous tentez de compléter une demande de fusion, vous serez refusé. Seules quelques personnes de notre équipe +sont autorisées à fusionner dans nos référentiels. -We check for new pull requests every day and will merge them in once they have been approved by someone in our team. \ No newline at end of file +Nous vérifions les nouvelles demande de fusion tous les jours et les fusionnerons une fois qu'elles auront été approuvées +par quelqu'un de notre équipe. diff --git a/Dockerfile b/Dockerfile index 928a479..8b1b5a3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,19 +1,35 @@ ARG branch=latest FROM cccs/assemblyline-v4-service-base:$branch -ENV SERVICE_PATH metadefender.MetaDefender +# Python path to the service class from your service directory +ENV SERVICE_PATH metadefender.metadefender.MetaDefender -# Switch to assemblyline user +# Install apt dependencies +USER root +COPY pkglist.txt /tmp/setup/ +RUN apt-get update && \ + apt-get upgrade -y && \ + apt-get install -y --no-install-recommends \ + $(grep -vE "^\s*(#|$)" /tmp/setup/pkglist.txt | tr "\n" " ") && \ + rm -rf /tmp/setup/pkglist.txt /var/lib/apt/lists/* + +# Install python dependencies USER assemblyline +COPY requirements.txt requirements.txt +RUN pip install \ + --no-cache-dir \ + --user \ + --requirement requirements.txt && \ + rm -rf ~/.cache/pip -# Copy MetaDefender service code +# Copy service code WORKDIR /opt/al_service COPY . . # Patch version in manifest -ARG version=4.0.0.dev1 +ARG version=1.0.0.dev1 USER root RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml # Switch to assemblyline user -USER assemblyline \ No newline at end of file +USER assemblyline diff --git a/LICENCE.md b/LICENCE.md deleted file mode 100644 index fd4ad66..0000000 --- a/LICENCE.md +++ /dev/null @@ -1,11 +0,0 @@ -MIT License - -Copyright (c) 2020 Crown Copyright, Government of Canada (Canadian Centre for Cyber Security / Communications Security Establishment) - -Copyright title to all 3rd party software distributed with Assemblyline (AL) is held by the respective copyright holders as noted in those files. Users are asked to read the 3rd Party Licenses referenced with those assets. - -Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..66eed2b --- /dev/null +++ b/LICENSE @@ -0,0 +1,27 @@ +MIT License + +Copyright (c) 2025 Crown Copyright, Government of Canada +(Canadian Centre for Cyber Security / Communications Security Establishment) + +Copyright title to all 3rd party software distributed with Assemblyline (AL) +is held by the respective copyright holders as noted in those files. Users +are asked to read the 3rd Party Licenses referenced with those assets. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d19eb32 --- /dev/null +++ b/Makefile @@ -0,0 +1,26 @@ +ifndef VERSION +$(error VERSION is undefined) +endif + +TAG?=latest +ORG?=cccs + +ifneq ($(ORG)x, x) +ORG:=$(ORG)/ +endif +ifneq ($(REGISTRY)x, x) +ORG:=$(REGISTRY)/ +endif + +.PHONY: default +default: build + +.PHONY: build +build: + docker build \ + --pull \ + --build-arg version=$(VERSION) \ + --build-arg branch=stable \ + -t $(REGISTRY)$(ORG)assemblyline-service-metadefender:$(TAG)\ + -f ./Dockerfile \ + . diff --git a/README.md b/README.md index 0594807..61b2827 100644 --- a/README.md +++ b/README.md @@ -1,39 +1,46 @@ +[![Discord](https://img.shields.io/badge/chat-on%20discord-7289da.svg?sanitize=true)](https://discord.gg/GUAy9wErNu) +[![](https://img.shields.io/discord/908084610158714900)](https://discord.gg/GUAy9wErNu) +[![Static Badge](https://img.shields.io/badge/github-assemblyline-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline) +[![Static Badge](https://img.shields.io/badge/github-assemblyline\_service\_metadefender-blue?logo=github)](https://github.com/CybercentreCanada/assemblyline-service-metadefender) +[![GitHub Issues or Pull Requests by label](https://img.shields.io/github/issues/CybercentreCanada/assemblyline/service-metadefender)](https://github.com/CybercentreCanada/assemblyline/issues?q=is:issue+is:open+label:service-metadefender) +[![License](https://img.shields.io/github/license/CybercentreCanada/assemblyline-service-metadefender)](./LICENSE) # MetaDefender Service This Assemblyline service interfaces with the [MetaDefender Core](https://www.opswat.com/metadefender-core) multi-scanning AV engine. +## Service Details **NOTE**: This service **requires you to buy** a licence. It also **requires you to install** MetaDefender Core on a seperate machine/VM. It is **not** preinstalled during a default installation. -## Overview +### Overview The MetaDefender service uses the MetaDefender Core API to send files to the MetaDefender Core server that you set-up to scan files for malware using upto 30 leading antivirus engines (depending on your license). The scan results from each of the installed antivirus engines are retrieved and displayed to the user. This service supports the use of multiple MetaDefender Core deployments for environments with heavy file loads. -## Licensing +### Licensing Contact your MetaDefender Core reseller to get access to the licence you need for your deployment: [https://www.opswat.com/partners/channel-partners#find-a-partner](https://www.opswat.com/partners/channel-partners#find-a-partner) -## Installing MetaDefender Core +### Installing MetaDefender Core **NOTE**: The following instructions are for **MetaDefender Core v4** running on a **Windows** machine. -1. Download the MetaDefender Core v4 installation package from the [OPSWAT Portal](https://portal.opswat.com/) +1. Download the MetaDefender Core v4 installation package from the [OPSWAT Portal](https://portal.opswat.com/) 2. Install MetaDefender Core v4 by following the instructions on the install wizard 3. Open a web browser and go to ``http://localhost:8008`` 4. Complete the basic configuration wizard to activate MetaDefender Core -## Configuring MetaDefender Core +### Configuring MetaDefender Core Once MetaDefender Core has been installed and activated with your license, the following configurations are recommended to improve the file scanning rate: * Using RAMDISK for the _tempdirectory_, see [here](https://onlinehelp.opswat.com/corev4/2.6._Special_installation_options.html) for instructions * Turning off the following engines under **Inventory > Technologies** - * Data sanitization engine + * Data sanitization engine * Archive engine * Frequently cleaning up the scan database using both of the following methods: * Setting all the data retention options to the lowest time value under **Settings > Data Retention** * Updating your MetaDefender Core version so that PostgreSQL is the default database -## Service Options +### Service Options * **api_key**: API Key used to connect to the MetaDefender API * **base_url**: The URL(s) of the MetaDefender deployment(s) @@ -48,14 +55,141 @@ Once MetaDefender Core has been installed and activated with your license, the f * **kw_score_revision_map**: A dictionary where the keys are the keywords that could be found in signatures, and the value is the revised score * **sig_score_revision_map**: A dictionary where the keys are the signatures that you want to revise, and the values are the scores that the signatures will be revised to -## Updating Antivirus Definitions +### Updating Antivirus Definitions Most of the antivirus vendors release definition updates at least once per day. Many release multiple daily. Some vendors release updates on weekends while others do not. Based on your type of deployment, you can select the frequency at which updates are applied. -### Online Deployment of MetaDefender Core +#### Online Deployment of MetaDefender Core If your MetaDefender Core is deployed in an online environment, you can set the update options by going to **Settings > Updates Settings**. You can also manually initiate an update by going to **Inventory > Technologies** and then clicking **UPDATE ALL**. -### Offline Deployment of MetaDefender Core +#### Offline Deployment of MetaDefender Core -If your MetaDefender Core is deployed in an offline environment, you will need to use the Update Downloader utility to download the antivirus definition updates in an online environment and then transfer the updates manually to the offline environment. See [here](https://onlinehelp.opswat.com/downloader/) for instructions on how to use the Update Downloader utility. Once the definition updates have been downloaded and transferred to the offline deployment, you can have MetaDefender monitor a local directory for any new definition updates added to it. You can set which local folder MetaDefender monitors by going to **Settings > Update Settings** then selecting **FOLDER** as the source for updates and then setting the **Pick up updates from** field to your local updates directory. \ No newline at end of file +If your MetaDefender Core is deployed in an offline environment, you will need to use the Update Downloader utility to download the antivirus definition updates in an online environment and then transfer the updates manually to the offline environment. See [here](https://onlinehelp.opswat.com/downloader/) for instructions on how to use the Update Downloader utility. Once the definition updates have been downloaded and transferred to the offline deployment, you can have MetaDefender monitor a local directory for any new definition updates added to it. You can set which local folder MetaDefender monitors by going to **Settings > Update Settings** then selecting **FOLDER** as the source for updates and then setting the **Pick up updates from** field to your local updates directory. + +## Image variants and tags + +Assemblyline services are built from the [Assemblyline service base image](https://hub.docker.com/r/cccs/assemblyline-v4-service-base), +which is based on Debian 11 with Python 3.11. + +Assemblyline services use the following tag definitions: + +| **Tag Type** | **Description** | **Example Tag** | +| :----------: | :----------------------------------------------------------------------------------------------- | :------------------------: | +| latest | The most recent build (can be unstable). | `latest` | +| build_type | The type of build used. `dev` is the latest unstable build. `stable` is the latest stable build. | `stable` or `dev` | +| series | Complete build details, including version and build type: `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | + +## Running this service + +This is an Assemblyline service. It is designed to run as part of the Assemblyline framework. + +If you would like to test this service locally, you can run the Docker image directly from the a shell: + + docker run \ + --name MetaDefender \ + --env SERVICE_API_HOST=http://`ip addr show docker0 | grep "inet " | awk '{print $2}' | cut -f1 -d"/"`:5003 \ + --network=host \ + cccs/assemblyline-service-metadefender + +To add this service to your Assemblyline deployment, follow this +[guide](https://cybercentrecanada.github.io/assemblyline4_docs/developer_manual/services/run_your_service/#add-the-container-to-your-deployment). + +## Documentation + +General Assemblyline documentation can be found at: https://cybercentrecanada.github.io/assemblyline4_docs/ + +# Service MetaDefender + +Ce service Assemblyline s'interface avec le moteur AV à balayage multiple de [MetaDefender Core] (https://www.opswat.com/metadefender-core). + +## Détails du service +**NOTE** : Ce service **nécessite l'achat** d'une licence. Il **exige également que vous installiez** MetaDefender Core sur une machine/VM séparée. Il n'est **pas** préinstallé lors d'une installation par défaut. + +### Vue d'ensemble + +Le service MetaDefender utilise l'API de MetaDefender Core pour envoyer des fichiers au serveur MetaDefender Core que vous avez configuré pour analyser les fichiers à la recherche de malwares en utilisant jusqu'à 30 moteurs antivirus de premier plan (en fonction de votre licence). Les résultats de l'analyse de chacun des moteurs antivirus installés sont récupérés et affichés à l'utilisateur. Ce service prend en charge l'utilisation de plusieurs déploiements de MetaDefender Core pour les environnements avec de lourdes charges de fichiers. + +### Licences + +Contactez votre revendeur MetaDefender Core pour obtenir la licence dont vous avez besoin pour votre déploiement : [https://www.opswat.com/partners/channel-partners#find-a-partner](https://www.opswat.com/partners/channel-partners#find-a-partner) + +### Installation de MetaDefender Core + +**REMARQUE** : Les instructions suivantes concernent **MetaDefender Core v4** fonctionnant sur une machine **Windows**. + +1. Téléchargez le paquet d'installation de MetaDefender Core v4 à partir du [Portail OPSWAT] (https://portal.opswat.com/). +2. Installez MetaDefender Core v4 en suivant les instructions de l'assistant d'installation. +3. Ouvrez un navigateur web et allez sur `http://localhost:8008``. +4. Complétez l'assistant de configuration de base pour activer MetaDefender Core. + +### Configurer MetaDefender Core + +Une fois que MetaDefender Core a été installé et activé avec votre licence, les configurations suivantes sont recommandées pour améliorer le taux d'analyse des fichiers : + +* Utilisation de RAMDISK pour le répertoire _tempdirectory_, voir [ici] (https://onlinehelp.opswat.com/corev4/2.6._Special_installation_options.html) pour les instructions. +* Désactiver les moteurs suivants sous **Inventaire > Technologies** + * Moteur d'assainissement des données + * Moteur d'archivage +* Nettoyer fréquemment la base de données d'analyse en utilisant les deux méthodes suivantes : + * Régler toutes les options de rétention des données à la valeur la plus basse sous **Paramètres > Rétention des données**. + * Mise à jour de la version de MetaDefender Core pour que PostgreSQL soit la base de données par défaut. + +### Options de service + +**api_key** : Clé API utilisée pour se connecter à l'API de MetaDefender +* **base_url** : L'URL du (des) déploiement(s) de MetaDefender + * Si vous avez un **seul** déploiement de MetaDefender Core, réglez la variable de service sur le type **str** et entrez l'URL de votre déploiement de MetaDefender Core. + * Si vous avez **plusieurs** déploiements de MetaDefender Core, définissez la variable de service comme étant de type **list** et entrez les URL de vos déploiements de MetaDefender Core en les séparant par une virgule. +**verify_certificate** : La valeur False ignore la vérification du certificat SSL. +**md_version** : Version de MetaDefender à laquelle vous vous connectez (3 ou 4) +**md_timeout** : Temps d'attente maximum lors de la connexion au serveur MetaDefender. +**max_md_scan_time** : Durée maximale d'attente des résultats de l'analyse avant que le serveur MetaDefender ne soit mis en attente (uniquement applicable lorsque plusieurs déploiements de MetaDefender sont utilisés). +**av_config** : Dictionnaire contenant les détails que nous utiliserons pour réviser ou omettre les signatures antivirus. + **blocklist** : Une liste d'éditeurs d'antivirus que nous voulons exclure de tous les résultats. + **kw_score_revision_map** : Un dictionnaire dont les clés sont les mots-clés qui pourraient être trouvés dans les signatures, et la valeur est le score révisé. + * **sig_score_revision_map** : Un dictionnaire dont les clés sont les signatures que vous souhaitez réviser et les valeurs sont les scores auxquels les signatures seront révisées. + +### Mise à jour des définitions d'antivirus + +La plupart des éditeurs d'antivirus publient des mises à jour de définitions au moins une fois par jour. Nombre d'entre eux le font plusieurs fois par jour. Certains éditeurs publient des mises à jour le week-end, d'autres non. En fonction de votre type de déploiement, vous pouvez sélectionner la fréquence à laquelle les mises à jour sont appliquées. + +#### Déploiement en ligne de MetaDefender Core + +Si votre MetaDefender Core est déployé dans un environnement en ligne, vous pouvez définir les options de mise à jour en allant dans **Paramètres > Paramètres des mises à jour**. Vous pouvez également lancer manuellement une mise à jour en allant dans **Inventaire > Technologies** puis en cliquant sur **MISE A JOUR TOUTES**. + +#### Déploiement hors ligne de MetaDefender Core + +Si votre MetaDefender Core est déployé dans un environnement hors ligne, vous devrez utiliser l'utilitaire Update Downloader pour télécharger les mises à jour des définitions de l'antivirus dans un environnement en ligne, puis transférer manuellement les mises à jour dans l'environnement hors ligne. Voir [ici] (https://onlinehelp.opswat.com/downloader/) pour savoir comment utiliser l'utilitaire Update Downloader. Une fois que les mises à jour des définitions ont été téléchargées et transférées vers le déploiement hors ligne, vous pouvez demander à MetaDefender de surveiller un répertoire local pour détecter toute nouvelle mise à jour de définition qui y serait ajoutée. Vous pouvez définir le dossier local que MetaDefender surveille en allant dans **Paramètres > Paramètres de mise à jour**, en sélectionnant **DOSSIER** comme source des mises à jour et en définissant le champ **Recueillir les mises à jour à partir de** sur votre répertoire de mises à jour local. + +## Variantes et étiquettes d'image + +Les services d'Assemblyline sont construits à partir de l'image de base [Assemblyline service](https://hub.docker.com/r/cccs/assemblyline-v4-service-base), +qui est basée sur Debian 11 avec Python 3.11. + +Les services d'Assemblyline utilisent les définitions d'étiquettes suivantes: + +| **Type d'étiquette** | **Description** | **Exemple d'étiquette** | +| :------------------: | :------------------------------------------------------------------------------------------------------------- | :------------------------: | +| dernière version | La version la plus récente (peut être instable). | `latest` | +| build_type | Type de construction utilisé. `dev` est la dernière version instable. `stable` est la dernière version stable. | `stable` ou `dev` | +| série | Détails de construction complets, comprenant la version et le type de build: `version.buildType`. | `4.5.stable`, `4.5.1.dev3` | + +## Exécution de ce service + +Ce service est spécialement optimisé pour fonctionner dans le cadre d'un déploiement d'Assemblyline. + +Si vous souhaitez tester ce service localement, vous pouvez exécuter l'image Docker directement à partir d'un terminal: + + docker run \ + --name MetaDefender \ + --env SERVICE_API_HOST=http://`ip addr show docker0 | grep "inet " | awk '{print $2}' | cut -f1 -d"/"`:5003 \ + --network=host \ + cccs/assemblyline-service-metadefender + +Pour ajouter ce service à votre déploiement d'Assemblyline, suivez ceci +[guide](https://cybercentrecanada.github.io/assemblyline4_docs/fr/developer_manual/services/run_your_service/#add-the-container-to-your-deployment). + +## Documentation + +La documentation générale sur Assemblyline peut être consultée à l'adresse suivante: https://cybercentrecanada.github.io/assemblyline4_docs/ diff --git a/metadefender/__init__.py b/metadefender/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/metadefender.py b/metadefender/metadefender.py similarity index 100% rename from metadefender.py rename to metadefender/metadefender.py diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..77877b4 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,2 @@ +[tool.cruft] +skip = ["pkglist.txt", "README.md", "metadefender", "tests"] diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..ff91064 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,2 @@ +assemblyline +assemblyline-v4-service diff --git a/service_manifest.yml b/service_manifest.yml index 4929a26..4c089d1 100644 --- a/service_manifest.yml +++ b/service_manifest.yml @@ -1,21 +1,25 @@ name: MetaDefender version: $SERVICE_TAG -description: > - Scan your files with a multi antivirus scanning engine. +description: This Assemblyline service interfaces with the [MetaDefender Core](https://www.opswat.com/metadefender-core) multi-scanning AV engine. +# Regex defining the types of files the service accepts and rejects accepts: .* rejects: empty|metadata/.* +# At which stage the service should run (one of FILTER, EXTRACT, CORE, SECONDARY, POST, REVIEW) +# NOTE: Stages are executed in the order defined in the list stage: CORE +# Which category the service is part of (one of Antivirus, Dynamic Analysis, External, Extraction, Filtering, Internet Connected, Networking, Static Analysis) category: Antivirus +# Does the service require access to the file to perform its task +# If set to false, the service will only have access to the file metadata (e.g. Hashes, size, type, ...) file_required: true +# Maximum execution time the service has before it's considered to be timed out timeout: 60 -disable_cache: false +# is the service enabled by default enabled: false -is_external: false -licence_count: 0 config: api_key: "" diff --git a/tests/gentests.py b/tests/gentests.py new file mode 100755 index 0000000..7ae6dde --- /dev/null +++ b/tests/gentests.py @@ -0,0 +1,30 @@ +#!/bin/env python +import os + +from assemblyline.common.importing import load_module_by_path +from assemblyline_service_utilities.testing.helper import TestHelper + +cwd = os.getcwd() +# Force manifest location +os.environ["SERVICE_MANIFEST_PATH"] = os.path.join(cwd, "service_manifest.yml") + +# Setup folder locations +RESULTS_FOLDER = os.path.join(cwd, "tests", "results") +SAMPLES_FOLDER = os.path.join(cwd, "tests", "samples") + +# Find which module we're working on +module = os.environ.get("SERVICE_PATH") +if not module: + for line in open("Dockerfile", "r").readlines(): + if line.startswith("ENV SERVICE_PATH"): + module = line[17:].strip() + break + +# Initialize test helper +service_class = load_module_by_path(module, cwd) +if os.path.exists(SAMPLES_FOLDER): + th = TestHelper(service_class, RESULTS_FOLDER, SAMPLES_FOLDER) +else: + th = TestHelper(service_class, RESULTS_FOLDER) + +th.regenerate_results(save_files=False) diff --git a/tests/gentests.sh b/tests/gentests.sh new file mode 100755 index 0000000..4759a1c --- /dev/null +++ b/tests/gentests.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +docker build \ + --pull \ + --build-arg branch=stable \ + -t ${PWD##*/}:gentests \ + -f ./Dockerfile \ + . + +if [[ -n "$FULL_SAMPLES_LOCATION" ]]; then + MOUNT_SAMPLES="-v ${FULL_SAMPLES_LOCATION}:/opt/samples" + ENV_SAMPLES="-e FULL_SAMPLES_LOCATION=/opt/samples" +fi +docker run \ + -t \ + --rm \ + -e FULL_SELF_LOCATION=/opt/al_service \ + $ENV_SAMPLES \ + -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla \ + -v $(pwd)/tests/:/opt/al_service/tests/ \ + $MOUNT_SAMPLES \ + ${PWD##*/}:gentests \ + bash -c "pip install -U -r tests/requirements.txt; python /opt/al_service/tests/gentests.py" diff --git a/tests/pytest.sh b/tests/pytest.sh new file mode 100755 index 0000000..66655c3 --- /dev/null +++ b/tests/pytest.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail + +docker build \ + --pull \ + --build-arg branch=stable \ + -t ${PWD##*/}:pytest \ + -f ./Dockerfile \ + . + +if [[ -n "$FULL_SAMPLES_LOCATION" ]]; then + MOUNT_SAMPLES="-v ${FULL_SAMPLES_LOCATION}:/opt/samples" + ENV_SAMPLES="-e FULL_SAMPLES_LOCATION=/opt/samples" +fi +docker run \ + -t \ + --rm \ + -e FULL_SELF_LOCATION=/opt/al_service \ + $ENV_SAMPLES \ + -v /usr/share/ca-certificates/mozilla:/usr/share/ca-certificates/mozilla \ + -v $(pwd)/tests/:/opt/al_service/tests/ \ + $MOUNT_SAMPLES \ + ${PWD##*/}:pytest \ + bash -c "pip install -U -r tests/requirements.txt; pytest -p no:cacheprovider --durations=10 -rsx -vv -x" diff --git a/test/requirements.txt b/tests/requirements.txt similarity index 54% rename from test/requirements.txt rename to tests/requirements.txt index 95f3075..c592336 100644 --- a/test/requirements.txt +++ b/tests/requirements.txt @@ -1,3 +1,5 @@ +assemblyline +assemblyline-service-utilities pytest requests-mock pytest-mock diff --git a/test/samples/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8 b/tests/samples/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8 similarity index 100% rename from test/samples/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8 rename to tests/samples/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8 diff --git a/test/test_metadefender.py b/tests/test_metadefender.py similarity index 99% rename from test/test_metadefender.py rename to tests/test_metadefender.py index ac80003..71966f8 100644 --- a/test/test_metadefender.py +++ b/tests/test_metadefender.py @@ -92,7 +92,7 @@ def remove_tmp_manifest(): def metadefender_class_instance(mocker, dummy_api_interface): create_tmp_manifest() try: - from metadefender import MetaDefender + from metadefender.metadefender import MetaDefender mocker.patch.object(MetaDefender, "get_api_interface", return_value=dummy_api_interface) yield MetaDefender() finally: @@ -125,7 +125,7 @@ def test_init(mocker): from json import dumps from assemblyline_v4_service.common.result import BODY_FORMAT, ResultSection mocker.patch("assemblyline_v4_service.common.api.ServiceAPIError") - from metadefender import AvHitSection + from metadefender.metadefender import AvHitSection av_name = "blah" virus_name = "blah" engine = {} @@ -208,7 +208,7 @@ def teardown_class(cls): def test_init(mocker): from assemblyline_v4_service.common.result import ResultSection mocker.patch("assemblyline_v4_service.common.api.ServiceAPIError") - from metadefender import AvErrorSection + from metadefender.metadefender import AvErrorSection av_name = "blah" engine = {} actual_res_sec = AvErrorSection(av_name, engine) diff --git a/tests/test_metadefender_samples.py b/tests/test_metadefender_samples.py new file mode 100644 index 0000000..11d3d6e --- /dev/null +++ b/tests/test_metadefender_samples.py @@ -0,0 +1,24 @@ +import os +import time + +import pytest +from assemblyline.common.importing import load_module_by_path +from assemblyline_service_utilities.testing.helper import TestHelper + +# Force manifest location +os.environ["SERVICE_MANIFEST_PATH"] = os.path.join(os.path.dirname(__file__), "..", "service_manifest.yml") + +# Setup folder locations +RESULTS_FOLDER = os.path.join(os.path.dirname(__file__), "results") +SAMPLES_FOLDER = os.path.join(os.path.dirname(__file__), "samples") + +# Initialize test helper +service_class = load_module_by_path("metadefender.metadefender.MetaDefender", os.path.join(os.path.dirname(__file__), "..")) +th = TestHelper(service_class, RESULTS_FOLDER, SAMPLES_FOLDER) + + +@pytest.mark.parametrize("sample", []) +def test_sample(sample): + start_time = time.time() + th.run_test_comparison(sample) + print(f"Time elapsed for {sample}: {round(time.time() - start_time)}s")