Skip to content

Commit

Permalink
Set IVRE's backend from the config
Browse files Browse the repository at this point in the history
  • Loading branch information
@p-l-
p-l- committed Jan 13, 2021
1 parent 30eb542 commit d4c8dc5
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 25 deletions.
27 changes: 27 additions & 0 deletions analyzers/IVRE/IVRE.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,33 @@
"multi": false,
"required": true,
"defaultValue": true
},
{
"name": "db_url",
"description": "The URL of the IVRE database (e.g., mongodb://host/ivre or http://host/cgi); defaults to using IVRE's configuration",
"type": "string",
"multi": false,
"required": false
},
{
"name": "db_url_data",
"description": "The URL of the IVRE database for the data purpose (e.g., maxmind:///usr/share/ivre/geoip or http://host/cgi); defaults to using IVRE's configuration",
"type": "string",
"multi": false,
"required": false
{
"name": "db_url_passive",
"description": "The URL of the IVRE database for the passive purpose (e.g., mongodb://host/ivre or http://host/cgi); defaults to using IVRE's configuration",
"type": "string",
"multi": false,
"required": false
},
{
"name": "db_url_scans",
"description": "The URL of the IVRE database for the scans (nmap) purpose (e.g., mongodb://host/ivre or http://host/cgi); defaults to using IVRE's configuration",
"type": "string",
"multi": false,
"required": false
}
],
"config": {
Expand Down
80 changes: 55 additions & 25 deletions analyzers/IVRE/ivre_analyzer.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,18 +13,19 @@


from cortexutils.analyzer import Analyzer
from ivre.db import db
from ivre import config
from ivre.db import MetaDB
from ivre import utils


# TODO: extracted data should depend on the data type!


DATABASES = {
"data": db.data,
"passive": db.passive,
"scans": db.nmap,
}
DATABASES = [
("data", "data"),
("passive", "passive"),
("scans", "nmap"),
]


class Processor:
Expand All @@ -51,7 +52,8 @@ def run(self):
def get(self, dbase):
# By default, report all IP addresses
result = sorted(
DATABASES[dbase].distinct("addr", flt=self.flt(dbase)), key=utils.ip2int
self.analyzer.databases[dbase].distinct("addr", flt=self.flt(dbase)),
key=utils.ip2int,
)
self.analyzer._artifacts.update(("ip", addr) for addr in result)
return result
Expand Down Expand Up @@ -82,14 +84,14 @@ class ProcessorIp(Processor):
keep_addresses = False

def flt(self, dbase):
return DATABASES[dbase].searchhost(self.data)
return self.analyzer.databases[dbase].searchhost(self.data)

def get_scans(self, dbase):
if self.keep_addresses:
all_results = {}
else:
result = {}
for rec in DATABASES[dbase].get(self.flt(dbase)):
for rec in self.analyzer.databases[dbase].get(self.flt(dbase)):
if self.keep_addresses:
result = all_results.setdefault(rec["addr"], {})
self.analyzer._artifacts.add(("ip", rec["addr"]))
Expand All @@ -110,7 +112,7 @@ def get_scans(self, dbase):
c for c in rec["categories"] if not c.startswith("_")
)
if rec.get("source"):
result.setdefault("source", set()).add(rec["source"])
result.setdefault("sources", set()).add(rec["source"])
if rec.get("hostnames"):
result.setdefault("hostnames", set()).update(
hn["name"] for hn in rec["hostnames"]
Expand Down Expand Up @@ -148,7 +150,7 @@ def get_passive(self, dbase):
all_results = {}
else:
result = {}
for rec in DATABASES[dbase].get(self.flt(dbase)):
for rec in self.analyzer.databases[dbase].get(self.flt(dbase)):
if self.keep_addresses:
result = all_results.setdefault(rec["addr"], {})
firstseen = rec.get("firstseen", rec.get("lastseen"))
Expand All @@ -166,7 +168,7 @@ def get_passive(self, dbase):
recontype = rec["recontype"]
result.setdefault("categories", set()).add(recontype)
if rec.get("sensor"):
result.setdefault("source", set()).add(rec["sensor"])
result.setdefault("sources", set()).add(rec["sensor"])
if recontype == "DNS_ANSWER":
result.setdefault("hostnames", set()).add(rec["value"])
self.analyzer._artifacts.add(("fqdn", rec["value"]))
Expand Down Expand Up @@ -241,7 +243,7 @@ def clean_results(result):

def get(self, dbase):
if dbase == "data":
result = DATABASES[dbase].infos_byip(self.data)
result = self.analyzer.databases[dbase].infos_byip(self.data)
if "as_num" in result:
self.analyzer._artifacts.add(("autonomous-system", result["as_num"]))
return result
Expand All @@ -263,7 +265,7 @@ class ProcessorNet(ProcessorIp):
keep_addresses = True

def flt(self, dbase):
return DATABASES[dbase].searchnet(self.data)
return self.analyzer.databases[dbase].searchnet(self.data)


class ProcessorAsnum(ProcessorNet):
Expand All @@ -274,7 +276,7 @@ class ProcessorAsnum(ProcessorNet):
"""

def flt(self, dbase):
return DATABASES[dbase].searchasnum(int(self.data))
return self.analyzer.databases[dbase].searchasnum(int(self.data))


class ProcessorPort(Processor):
Expand All @@ -289,16 +291,16 @@ def flt(self, dbase):
proto = "tcp"
port = int(self.data)
if proto != "udp" and dbase == "passive":
return DATABASES[dbase].searchnonexistent()
return DATABASES[dbase].searchport(port, protocol=proto)
return self.analyzer.databases[dbase].searchnonexistent()
return self.analyzer.databases[dbase].searchport(port, protocol=proto)


class ProcessorCert(Processor):

databases = ["passive", "scans"]

def flt(self, dbase):
return DATABASES[dbase].searchcert(
return self.analyzer.databases[dbase].searchcert(
**{{32: "md5", 40: "sha1", 64: "sha256"}[len(self.data)]: self.data}
)

Expand All @@ -309,18 +311,18 @@ class ProcessorFqdn(Processor):

def flt(self, dbase):
if dbase == "passive":
return DATABASES[dbase].searchdns(name=self.data)
return DATABASES[dbase].searchhostname(self.data)
return self.analyzer.databases[dbase].searchdns(name=self.data)
return self.analyzer.databases[dbase].searchhostname(self.data)

def rev_flt(self, dbase):
return DATABASES[dbase].searchdns(name=self.data, reverse=True)
return self.analyzer.databases[dbase].searchdns(name=self.data, reverse=True)

def get(self, dbase):
if dbase == "passive":
# specific case: two filters (w/ & w/o reverse=True)
addresses = set()
names = set()
for rec in DATABASES[dbase].get(
for rec in self.analyzer.databases[dbase].get(
self.flt(dbase), fields=["addr", "value", "targetval"]
):
if rec.get("addr"):
Expand All @@ -339,11 +341,15 @@ def get(self, dbase):
class ProcessorDomain(ProcessorFqdn):
def flt(self, dbase):
if dbase == "passive":
return DATABASES[dbase].searchdns(name=self.data, subdomains=True)
return DATABASES[dbase].searchdomain(self.data)
return self.analyzer.databases[dbase].searchdns(
name=self.data, subdomains=True
)
return self.analyzer.databases[dbase].searchdomain(self.data)

def rev_flt(self, dbase):
return DATABASES[dbase].searchdns(name=self.data, subdomains=True, reverse=True)
return self.analyzer.databases[dbase].searchdns(
name=self.data, subdomains=True, reverse=True
)


class ProcessorUserAgent(Processor):
Expand All @@ -358,6 +364,30 @@ class IVREAnalyzer(Analyzer):
def __init__(self):
Analyzer.__init__(self)
self._artifacts = set()
self.db = MetaDB(
self.get_param(
"config.db_url", default=config.DB if hasattr(config, "DB") else None
),
urls={
attr: url
for attr, url in (
(
attr,
self.get_param(
"config.db_url_%s" % name,
default=(
getattr(config, "DB_%s" % attr.upper())
if hasattr(config, "DB_%s" % attr.upper())
else None
),
),
)
for name, attr in DATABASES
)
if url
},
)
self.databases = {name: getattr(self.db, attr) for name, attr in DATABASES}

@staticmethod
def summary(raw):
Expand Down

0 comments on commit d4c8dc5

Please sign in to comment.