Merge pull request #1 from CybercentreCanada/staging_prep

Preparation to move to staging

Dockerfile

@@ -0,0 +1,18 @@
+FROM cccs/assemblyline-v4-service-base:latest
+
+ENV SERVICE_PATH avclass_.avclass_.AVclass
+
+# Switch to assemblyline user
+USER assemblyline
+
+# Copy service code
+WORKDIR /opt/al_service
+COPY . .
+
+# Patch version in manifest
+ARG version=4.0.0.dev1
+USER root
+RUN sed -i -e "s/\$SERVICE_TAG/$version/g" service_manifest.yml
+
+# Switch to assemblyline user
+USER assemblyline

avclass_/avclass_.py

@@ -8,7 +8,8 @@
 from assemblyline_v4_service.common.base import ServiceBase
 from assemblyline_v4_service.common.request import ServiceRequest
 from assemblyline_v4_service.common.result import Result, ResultSection, BODY_FORMAT, Heuristic
-from avclass.avclass2.lib.avclass2_common import SampleInfo, AvLabels
+
+from avclass_.avclass.avclass2.lib.avclass2_common import SampleInfo, AvLabels
 
 DATA_PATH = Path(resource_filename(__name__, 'data'))
 TAG_PATH = DATA_PATH / 'avclass.tagging'

pipelines/azure-build.yaml

@@ -0,0 +1,35 @@
+name: build
+
+trigger:
+  tags:
+    include: ["v*"]
+pr: none
+
+pool:
+  vmImage: 'ubuntu-18.04'
+
+stages:
+- stage: deploy
+  jobs:
+  - job: deploy
+    displayName: Deploy containers to dockerhub
+    variables:
+      - group: deployment-information
+    steps:
+    - task: Docker@2
+      displayName: Login to docker hub
+      inputs:
+        command: login
+        containerRegistry: dockerhub
+    - script: |
+        set -xv  # Echo commands before they are run
+        export TAG=${BUILD_SOURCEBRANCH#"refs/tags/v"}
+        if [[ "$TAG" == *stable* ]]; then export BUILD_TYPE=stable; else export BUILD_TYPE=latest; fi
+        docker build --build-arg version=$TAG -t cccs/assemblyline-service-avclass:$TAG -t cccs/assemblyline-service-avclass:$BUILD_TYPE .
+      displayName: Build containers
+    - script: |
+        docker run -v `pwd`/test/:/opt/al_service/test/ cccs/assemblyline-service-avclass:latest bash -c 'pip install -U -r test/requirements.txt; pytest'
+      displayName: Test containers
+    - script: |
+        docker push cccs/assemblyline-service-avclass --all-tags
+      displayName: Deploy to Docker Hub

pipelines/azure-tests.yaml

@@ -0,0 +1,37 @@
+name: tests
+
+trigger: ["*"]
+pr: ["*"]
+
+pool:
+  vmImage: 'ubuntu-18.04'
+
+jobs:
+- job: run_test
+  strategy:
+    matrix:
+      python3_7:
+        python.version: '3.7'
+      Python3_8:
+        python.version: '3.8'
+
+  timeoutInMinutes: 10
+
+  steps:
+    - task: UsePythonVersion@0
+      displayName: Set python version
+      inputs:
+        versionSpec: '$(python.version)'
+    - script: |
+        set -xv  # Echo commands before they are run
+        sudo apt-get update
+        sudo apt-get install -y libfuzzy-dev libfuzzy2
+        sudo rm -rf /var/lib/apt/lists/*
+        sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir assemblyline assemblyline_v4_service
+        sudo env "PATH=$PATH" python -m pip install -U --no-cache-dir -r `pwd`/test/requirements.txt
+        sudo rm -rf /tmp/* /var/lib/apt/lists/* ~/.cache/pip
+      displayName: Setup environment
+    - script: python -m pytest --durations=10 -rsx -vv --cov-report=xml --cov=avclass
+      displayName: Test
+    - script: python -m codecov
+      displayName: Upload Coverage

avclass_/service_manifest.yml → service_manifest.yml

@@ -1,7 +1,6 @@
 name: AVclass
-version: 1
-description: >
-  Extracts malware family and details from AV labels
+version: $SERVICE_TAG
+description: Extracts malware family and details from AV labels
 
 accepts: .*
 rejects: empty|metadata/.*
@@ -13,7 +12,7 @@ file_required: false
 timeout: 10
 disable_cache: false
 
-enabled: true
+enabled: false
 is_external: false
 licence_count: 0
 
@@ -44,6 +43,5 @@ heuristics:
     description: AV labels indicate PUP
 
 docker_config:
-  image: ${REGISTRY}testing/assemblyline-service-avclass:latest
-  cpu_cores: 1.0
-  ram_mb: 256
+  image: ${REGISTRY}cccs/assemblyline-service-avclass:$SERVICE_TAG
+  cpu_cores: 0.25

test/requirements.txt

@@ -0,0 +1,3 @@
+pytest
+pytest-cov
+codecov

test/results/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8.json

@@ -0,0 +1 @@
+{"classification": "TLP:W", "response": {"milestones": {"service_started": null, "service_completed": null}, "service_version": null, "service_name": "avclass", "service_tool_version": null, "supplementary": [], "extracted": [], "service_context": null, "service_debug_info": null}, "result": {"score": 0, "sections": [{"body": null, "body_format": "TEXT", "classification": "TLP:W", "depth": 0, "heuristic": null, "tags": {}, "title_text": "No AV labels"}]}, "sha256": "dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8", "drop_file": false, "temp_submission_data": {}}

test/samples/dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8

@@ -0,0 +1 @@
+this is a text file

test/test_service.py

@@ -0,0 +1,169 @@
+import os
+import json
+import pytest
+import shutil
+
+# Getting absolute paths, names and regexes
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+ROOT_DIR = os.path.dirname(TEST_DIR)
+SERVICE_CONFIG_NAME = "service_manifest.yml"
+SERVICE_CONFIG_PATH = os.path.join(ROOT_DIR, SERVICE_CONFIG_NAME)
+TEMP_SERVICE_CONFIG_PATH = os.path.join("/tmp", SERVICE_CONFIG_NAME)
+
+
+# Samples that we will be sending to the service
+samples = [
+    dict(
+        sid=1,
+        metadata={},
+        service_name='avclass',
+        service_config={},
+        fileinfo=dict(
+            magic='ASCII text, with no line terminators',
+            md5='fda4e701258ba56f465e3636e60d36ec',
+            mime='text/plain',
+            sha1='af2c2618032c679333bebf745e75f9088748d737',
+            sha256='dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8',
+            size=19,
+            type='unknown',
+        ),
+        filename='dadc624d4454e10293dbd1b701b9ee9f99ef83b4cd07b695111d37eb95abcff8',
+        min_classification='TLP:WHITE',
+        max_files=501,  # TODO: get the actual value
+        ttl=3600,
+    ),
+]
+
+
+def create_tmp_manifest():
+    temp_service_config_path = os.path.join("/tmp", SERVICE_CONFIG_NAME)
+    if not os.path.exists(temp_service_config_path):
+        # Placing the service_manifest.yml in the tmp directory
+        shutil.copyfile(SERVICE_CONFIG_PATH, temp_service_config_path)
+
+
+def remove_tmp_manifest():
+    temp_service_config_path = os.path.join("/tmp", SERVICE_CONFIG_NAME)
+    if os.path.exists(temp_service_config_path):
+        os.remove(temp_service_config_path)
+
+
+@pytest.fixture
+def avclass_class_instance():
+    create_tmp_manifest()
+    try:
+        from avclass_.avclass_ import AVclass
+        yield AVclass()
+    finally:
+        remove_tmp_manifest()
+
+
+def check_equality_of_named_tuples(this, that):
+    if this._fields == that._fields:
+        return True
+    else:
+        return False
+
+
+class TestModule:
+    @classmethod
+    def setup_class(cls):
+        create_tmp_manifest()
+
+    @classmethod
+    def teardown_class(cls):
+        remove_tmp_manifest()
+
+    @staticmethod
+    def test_path_constants():
+        from pathlib import Path
+        from avclass_.avclass_ import DATA_PATH, TAG_PATH, EXP_PATH, TAX_PATH
+        data_path = Path(os.path.join(os.path.dirname(os.getcwd()), "avclass_", "data"))
+        assert DATA_PATH == data_path
+        assert TAG_PATH == Path(os.path.join(data_path, "avclass.tagging"))
+        assert EXP_PATH == Path(os.path.join(data_path, "avclass.expansion"))
+        assert TAX_PATH == Path(os.path.join(data_path, "avclass.taxonomy"))
+
+    @staticmethod
+    def test_avclass_constants():
+        from collections import namedtuple
+        from avclass_.avclass_ import AVClassTag, AVClassTags, AVCLASS_CATEGORY
+        correct_avclass_tag = namedtuple('AVClassTag', ['name', 'path', 'category', 'rank'])
+        correct_avclass_tags = namedtuple('AVClassTags', ['tags', 'is_pup', 'family'])
+        assert check_equality_of_named_tuples(correct_avclass_tag, AVClassTag)
+        assert check_equality_of_named_tuples(correct_avclass_tags, AVClassTags)
+        assert AVCLASS_CATEGORY == {
+            'FAM': ('family', 1),
+            'BEH': ('behavior', 2),
+            'CLASS': ('classification', 3),
+            'FILE': ('file', 4),
+            'GEN': ('generic', None),
+            'UNK': ('unknown', None),
+        }
+
+
+class TestAVClass:
+    @classmethod
+    def setup_class(cls):
+        create_tmp_manifest()
+
+    @classmethod
+    def teardown_class(cls):
+        remove_tmp_manifest()
+
+    @staticmethod
+    def test_init(avclass_class_instance):
+        # Code coverage, yay!
+        assert True
+
+    @staticmethod
+    def test_start(avclass_class_instance):
+        # Code coverage, yay!
+        avclass_class_instance.start()
+        # TODO: somehow assert that avclass_class_instance._av_labels is correct
+        assert True
+
+    @staticmethod
+    @pytest.mark.parametrize("sample_info, expected_result", [(None, False)])
+    def test_get_avclass_tags(sample_info, expected_result, avclass_class_instance):
+        # TODO: write tests that verify that this method works correctly
+        assert True
+
+    @staticmethod
+    @pytest.mark.parametrize("sample", samples)
+    def test_execute(sample, avclass_class_instance):
+        # TODO: Break down the execute method to make it easily testable
+        from assemblyline_v4_service.common.task import Task
+        from assemblyline.odm.messages.task import Task as ServiceTask
+        from assemblyline_v4_service.common.request import ServiceRequest
+
+        service_task = ServiceTask(sample)
+        task = Task(service_task)
+        avclass_class_instance._task = task
+        service_request = ServiceRequest(task)
+
+        # Actually executing the sample
+        avclass_class_instance.execute(service_request)
+
+        # Get the result of execute() from the test method
+        test_result = task.get_service_result()
+
+        # Get the assumed "correct" result of the sample
+        correct_result_path = os.path.join(TEST_DIR, "results", task.file_name + ".json")
+        with open(correct_result_path, "r") as f:
+            correct_result = json.loads(f.read())
+        f.close()
+
+        # Assert that the appropriate sections of the dict are equal
+
+        # Avoiding unique items in the response
+        test_result_response = test_result.pop("response")
+        correct_result_response = correct_result.pop("response")
+        assert test_result == correct_result
+
+        # Comparing everything in the response except for the service_completed and the output.json supplementary
+        test_result_response["milestones"].pop("service_completed")
+        correct_result_response["milestones"].pop("service_completed")
+        correct_result_response.pop("supplementary")
+        test_result_response.pop("supplementary")
+        assert test_result_response == correct_result_response