WIP/evaluation: Using optionally Docker to prevent compatibility problems with Analyzers

.travis.yml

@@ -0,0 +1,10 @@
+sudo: required
+
+services:
+  - docker
+
+language: generic
+
+script:
+- bash create_docker_analyzers.sh
+- bash build_docker_images.sh

README.md

@@ -10,6 +10,19 @@ page for details on how to write and submit one.
 
 To configure analyzers, please read the [Cortex Analyzer Requirements Guide](https://github.com/TheHive-Project/CortexDocs/blob/master/analyzer_requirements.md).
 
+# Cortex-Analyzers and Docker
+[![Status](https://api.travis-ci.org/3c7/Cortex-Analyzers.svg?branch=features%2Fdockerized)](https://github.com/3c7/Cortex-Analyzers/tree/features/dockerized)
+
+This is in an early testing stage. Do not use the "dockerized" analyzers in production environments. To build the docker images, the script `./build_docker_images.sh` can be used which creates two base images (Python 2 and Python 3) based on alpine linux and iterates over the Dockerfiles available in analyzers-docker. To use the analyzers in Cortex, the path has to be changed from `analyzers` to `analyzers-docker`. Every analyzer runs in the respective docker container in interactive mode. Analyzer can be tested in the following way:
+```
+#               IMAGE NAME                      python(3)     python module
+$ docker run -i cortex-analyzers-abuse_finder   python        abuse_finder.py <<< '{
+"dataType": "domain",
+"data": "google.de"
+}'
+{<output here>}
+```
+
 # License
 **Unless specified otherwise**, analyzers are released under the [AGPL](https://github.com/TheHive-Project/Cortex-analyzers/blob/master/LICENSE) (Affero General Public License).
 

analyzers-docker/Abuse_Finder/Abuse_Finder.json

@@ -0,0 +1,30 @@
+{
+  "name": "Abuse_Finder",
+  "version": "2.0",
+  "author": "CERT-BDF",
+  "url": "https://github.com/CERT-BDF/Cortex-Analyzers",
+  "license": "AGPL-V3",
+  "description": "Find abuse contacts associated with domain names, URLs, IPs and email addresses.",
+  "dataTypeList": ["ip", "domain", "url", "mail"],
+  "command": "docker run -i cortex-analyzers-abuse_finder",
+  "baseConfig": "Abuse_Finder",
+  "config": {},
+  "configurationItems": [
+    {
+      "name": "check_tlp",
+      "description": "Define if the analyzer should check TLP of data before running",
+      "type": "boolean",
+      "multi": false,
+      "required": true,
+      "defaultValue": false
+    },
+    {
+      "name": "max_tlp",
+      "description": "Define the maximum TLP level autorized",
+      "type": "number",
+      "multi": false,
+      "required": true,
+      "defaultValue": 3
+    }
+  ]
+}

analyzers-docker/Abuse_Finder/Dockerfile

@@ -0,0 +1,5 @@
+FROM cortex-base-python2
+ADD ./analyzers/Abuse_Finder /analyzer
+WORKDIR /analyzer
+RUN pip install --no-cache-dir -r requirements.txt
+CMD ./abusefinder.py

analyzers-docker/Malpedia/Dockerfile

@@ -0,0 +1,5 @@
+FROM cortex-base-python3
+ADD ./analyzers/Malpedia /analyzer
+WORKDIR /analyzer
+RUN apk add build-base --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/base/ --allow-untrusted && apk add python3-dev && pip3 install --no-cache-dir -r requirements.txt && apk del build-base python3-dev
+CMD ./malpedia_analyzer.py

analyzers-docker/Malpedia/Malpedia.json

@@ -0,0 +1,34 @@
+{
+  "name": "Malpedia",
+  "author": "Davide Arcuri and Andrea Garavaglia, LDO-CERT",
+  "license": "AGPL-V3",
+  "url": "https://github.com/LDO-CERT/cortex-analyzers",
+  "version": "1.0",
+  "description": "Check files against Malpedia YARA rules.",
+  "dataTypeList": ["file"],
+  "baseConfig": "Malpedia",
+  "command": "docker run -i cortex-analyzers-malpedia",
+  "configurationItems": [
+    {
+      "name": "path",
+      "description": "Rulepath",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "username",
+      "description": "Username",
+      "type": "string",
+      "multi": false,
+      "required": true
+    },
+    {
+      "name": "password",
+      "description": "Password",
+      "type": "string",
+      "multi": false,
+      "required": true
+    }
+  ]
+}

analyzers-docker/MaxMind/Dockerfile

@@ -0,0 +1,5 @@
+FROM cortex-base-python3
+ADD ./analyzers/MaxMind /analyzer
+WORKDIR /analyzer
+RUN pip3 install --no-cache-dir -r requirements.txt
+CMD ./geo.py

analyzers-docker/MaxMind/MaxMind_GeoIP.json

@@ -0,0 +1,32 @@
+{
+    "name": "MaxMind_GeoIP",
+    "version": "3.0",
+    "author": "CERT-BDF",
+    "url": "https://github.com/CERT-BDF/Cortex-Analyzers",
+    "license": "AGPL-V3",
+    "description": "Use MaxMind to geolocate an IP address.",
+    "dataTypeList": ["ip"],
+    "command": "docker run -i cortex-analyzers-maxmind",
+    "baseconfig":"MaxMind_GeoIP",
+    "config": {
+        "service": ""
+    },
+    "configurationItems": [
+    {
+      "name": "check_tlp",
+      "description": "Define if the analyzer should check TLP of data before running",
+      "type": "boolean",
+      "multi": false,
+      "required": true,
+      "defaultValue": true
+    },
+    {
+      "name": "max_tlp",
+      "description": "Define the maximum TLP level autorized",
+      "type": "number",
+      "multi": false,
+      "required": true,
+      "defaultValue": 3
+    }
+  ]
+}

analyzers-docker/MsgParser/Dockerfile

@@ -0,0 +1,5 @@
+FROM cortex-base-python2
+ADD ./analyzers/MsgParser /analyzer
+WORKDIR /analyzer
+RUN pip install --no-cache-dir -r requirements.txt
+CMD ./parse.py

analyzers-docker/MsgParser/Msg_Parser.json

@@ -0,0 +1,34 @@
+{
+    "name": "Msg_Parser",
+    "version": "2.0",
+    "author": "CERT-BDF",
+    "url": "https://github.com/CERT-BDF/Cortex-Analyzers",
+    "license": "AGPL-V3",
+    "description": "Parse Outlook MSG files and extract the main artifacts.",
+    "dataTypeList": [
+        "file"
+    ],
+    "command": "docker run -i cortex-analyzers-msgparser",
+    "baseconfig":"Msg_Parser",
+    "config": {
+        "service": ""
+    },
+    "configurationItems": [
+    {
+      "name": "check_tlp",
+      "description": "Define if the analyzer should check TLP of data before running",
+      "type": "boolean",
+      "multi": false,
+      "required": true,
+      "defaultValue": true
+    },
+    {
+      "name": "max_tlp",
+      "description": "Define the maximum TLP level autorized",
+      "type": "number",
+      "multi": false,
+      "required": true,
+      "defaultValue": 3
+    }
+  ]
+}

analyzers-docker/Yara/Dockerfile

@@ -0,0 +1,5 @@
+FROM cortex-base-python3
+ADD ./analyzers/Yara /analyzer
+WORKDIR /analyzer
+RUN apk add build-base --update-cache --repository http://dl-3.alpinelinux.org/alpine/edge/base/ --allow-untrusted && apk add python3-dev && pip3 install --no-cache-dir -r requirements.txt && apk del build-base python3-dev
+CMD ./yara_analyzer.py

analyzers-docker/Yara/Yara.json

@@ -0,0 +1,38 @@
+{
+    "name": "Yara",
+    "author": "Nils Kuhnert, CERT-Bund",
+    "license": "AGPL-V3",
+    "url": "https://github.com/BSI-CERT-Bund/cortex-analyzers",
+    "version": "2.0",
+    "description": "Check files against YARA rules.",
+    "dataTypeList": ["file"],
+    "command": "docker run -i cortex-analyzers-yara",
+    "baseConfig": "Yara",
+    "config": {},
+    "configurationItems": [
+      {
+        "name": "check_tlp",
+        "description": "Define if the analyzer should check TLP of data before running",
+        "type": "boolean",
+        "multi": false,
+        "required": true,
+        "defaultValue": false
+      },
+      {
+        "name": "max_tlp",
+        "description": "Define the maximum TLP level autorized",
+        "type": "number",
+        "multi": false,
+        "required": true,
+        "defaultValue": 3
+    },
+    {
+      "name": "rules",
+      "description": "Define the path rules folder",
+      "type": "string",
+      "multi": false,
+      "required": true,
+      "defaultValue": "path"
+    }
+    ]
+}

analyzers/Cymon/Cymon_Check_IP.json

@@ -1,6 +1,6 @@
 {
     "name": "Cymon_Check_IP",
-    "version": "2.0",
+    "version": "2.1",
     "author": "Julian Gonzalez",
     "url": "https://github.com/ST2labs/Analyzers",
     "license": "AGPL-V3",

analyzers/JoeSandbox/JoeSandbox_File_Analysis_Inet.json

@@ -32,7 +32,7 @@
       "type": "number",
       "multi": false,
       "required": true,
-      "defaultValue": 240
+      "defaultValue": 1800
     },
     {
       "name": "networktimeout",

analyzers/JoeSandbox/JoeSandbox_File_Analysis_Noinet.json

@@ -32,7 +32,15 @@
       "type": "number",
       "multi": false,
       "required": true,
-      "defaultValue": 240
+      "defaultValue": 1800
+    },
+    {
+      "name": "networktimeout",
+      "description": "Network timeout (second)",
+      "type": "number",
+      "multi": false,
+      "required": true,
+      "defaultValue": 30
     }
   ]
 }

analyzers/JoeSandbox/JoeSandbox_Url_Analysis.json

@@ -32,7 +32,7 @@
       "type": "number",
       "multi": false,
       "required": true,
-      "defaultValue": 240
+      "defaultValue": 1800
     },
     {
       "name": "networktimeout",

build_docker_images.sh

@@ -0,0 +1,36 @@
+#!/usr/bin/env bash
+# Vars
+VER=1.0.0
+PREFIX=$1
+
+# Docker base images
+echo "Building base images..."
+if [ ! ${PREFIX} = "" ]; then
+  docker build -t ${PREFIX}/cortex-base-python2:${VER} --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python2
+  docker build -t ${PREFIX}/cortex-base-python2:latest --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python2
+  docker build -t ${PREFIX}/cortex-base-python3:${VER} --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python3
+  docker build -t ${PREFIX}/cortex-base-python3:latest --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python3
+else
+  docker build -t cortex-base-python2:${VER} --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python2
+  docker build -t cortex-base-python2:latest --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python2
+  docker build -t cortex-base-python3:${VER} --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python3
+  docker build -t cortex-base-python3:latest --build-arg VERSION=${VER} --build-arg BUILD_DATE=`date -u +"%Y-%m-%dT%H:%M:%SZ"` --build-arg VCS_REF=`git rev-parse --short HEAD` contrib/docker/cortex-base-python3
+fi
+
+echo "Building analyzer images..."
+for analyzer in `ls -1 analyzers-docker/`; do
+  lower=`echo ${analyzer} | tr "[:upper:]" "[:lower:]"`
+  echo "Building cortex-analyzers-$lower"
+  if [ ! ${PREFIX} = "" ]; then
+    OUTPUT=$(docker build -t ${PREFIX}/cortex-analyzers-${lower} -f analyzers-docker/${analyzer}/Dockerfile .)
+  else
+    OUTPUT=$(docker build -t cortex-analyzers-${lower} -f analyzers-docker/${analyzer}/Dockerfile .)
+  fi
+  if [ $? != 0 ]; then
+    echo -e "\e[91m$OUTPUT\e[0m"
+    echo -e "\e[100m\e[91mError while building image for $analyzer.\e[0m"
+    exit 1
+  fi
+done
+echo -e "\e[100m\e[92mSuccessfully built analyzer docker images. Check them via 'docker image ls'.\e[0m"
+

contrib/docker/cortex-base-python2/Dockerfile

@@ -0,0 +1,17 @@
+FROM alpine:latest
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+
+LABEL org.label-schema.build-date=$BUILD_DATE \
+          org.label-schema.name="Cortex-Analyzers Python2 base image" \
+          org.label-schema.description="This is the base image for using analyzers with python2." \
+          org.label-schema.url="https://thehive-project.org" \
+          org.label-schema.vcs-ref=$VCS_REF \
+          org.label-schema.vcs-url="e.g. https://github.com/TheHive-Project/Cortex-Analyzers" \
+          org.label-schema.vendor="TheHive-Project" \
+          org.label-schema.version=$VERSION \
+          org.label-schema.schema-version="1.0"
+
+RUN apk add --no-cache python py-pip git && pip install --upgrade --no-cache-dir pip cortexutils

contrib/docker/cortex-base-python3/Dockerfile

@@ -0,0 +1,17 @@
+FROM alpine:latest
+
+ARG BUILD_DATE
+ARG VCS_REF
+ARG VERSION
+
+LABEL org.label-schema.build-date=$BUILD_DATE \
+          org.label-schema.name="Cortex-Analyzers Python3 base image" \
+          org.label-schema.description="This is the base image for using analyzers with python3." \
+          org.label-schema.url="https://thehive-project.org" \
+          org.label-schema.vcs-ref=$VCS_REF \
+          org.label-schema.vcs-url="e.g. https://github.com/TheHive-Project/Cortex-Analyzers" \
+          org.label-schema.vendor="TheHive-Project" \
+          org.label-schema.version=$VERSION \
+          org.label-schema.schema-version="1.0"
+
+RUN apk add --no-cache python3 py-pip git && pip install --upgrade --no-cache-dir pip cortexutils