Skip to content

Commit

Permalink
#170 Add support of alerts Convert MISP into alert
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om authored and nadouani committed Apr 10, 2017
1 parent 3aa3707 commit 63adafe
Show file tree
Hide file tree
Showing 12 changed files with 861 additions and 492 deletions.
15 changes: 7 additions & 8 deletions thehive-backend/app/connectors/Connectors.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,14 @@ package connectors

import javax.inject.Inject

import scala.collection.immutable

import com.google.inject.AbstractModule
import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder }
import play.api.libs.json.{ JsObject, Json }
import play.api.mvc.{ Action, Results }
import play.api.routing.{ Router, SimpleRouter }
import play.api.routing.sird.UrlContext
import play.api.routing.{ Router, SimpleRouter }

import com.google.inject.AbstractModule

import net.codingwell.scalaguice.{ ScalaModule, ScalaMultibinder }
import scala.collection.immutable

trait Connector {
val name: String
Expand All @@ -20,10 +18,11 @@ trait Connector {
}

class ConnectorRouter @Inject() (connectors: immutable.Set[Connector]) extends SimpleRouter {
def get(connectorName: String): Option[Connector] = connectors.find(_.name == connectorName)

def routes = {
case request @ p"/$connector/$path<.*>"
connectors
.find(_.name == connector)
get(connector)
.flatMap(_.router.withPrefix(s"/$connector/").handlerFor(request))
.getOrElse(Action { _ Results.NotFound(s"connector $connector not found") })
}
Expand Down
91 changes: 91 additions & 0 deletions thehive-backend/app/controllers/Alert.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package controllers

import javax.inject.{ Inject, Singleton }

import akka.stream.Materializer
import org.elastic4play.controllers.{ Authenticated, FieldsBodyParser, Renderer }
import org.elastic4play.models.JsonFormat.baseModelEntityWrites
import org.elastic4play.services.JsonFormat.{ aggReads, queryReads }
import org.elastic4play.services._
import org.elastic4play.{ BadRequestError, Timed }
import play.api.Logger
import play.api.http.Status
import play.api.libs.json.JsArray
import play.api.mvc.Controller
import services.AlertSrv

import scala.concurrent.{ ExecutionContext, Future }
import scala.util.Try

@Singleton
class AlertCtrl @Inject() (
alertSrv: AlertSrv,
auxSrv: AuxSrv,
authenticated: Authenticated,
renderer: Renderer,
fieldsBodyParser: FieldsBodyParser,
implicit val ec: ExecutionContext,
implicit val mat: Materializer) extends Controller with Status {

val log = Logger(getClass)

@Timed
def create() = authenticated(Role.write).async(fieldsBodyParser) { implicit request
alertSrv.create(request.body)
.map(alert renderer.toOutput(CREATED, alert))
}

@Timed
def get(id: String) = authenticated(Role.read).async { implicit request
val withStats = for {
statsValues request.queryString.get("nstats")
firstValue statsValues.headOption
} yield Try(firstValue.toBoolean).getOrElse(firstValue == "1")

for {
alert alertSrv.get(id)
alertsWithStats auxSrv.apply(alert, 0, withStats.getOrElse(false), removeUnaudited = false)
} yield renderer.toOutput(OK, alertsWithStats)
}

@Timed
def update(id: String) = authenticated(Role.write).async(fieldsBodyParser) { implicit request
alertSrv.update(id, request.body)
.map { alert renderer.toOutput(OK, alert) }
}

@Timed
def bulkUpdate() = authenticated(Role.write).async(fieldsBodyParser) { implicit request
request.body.getStrings("ids").fold(Future.successful(Ok(JsArray()))) { ids
alertSrv.bulkUpdate(ids, request.body.unset("ids")).map(multiResult renderer.toMultiOutput(OK, multiResult))
}
}

@Timed
def delete(id: String) = authenticated(Role.write).async { implicit request
alertSrv.delete(id)
.map(_ NoContent)
}

@Timed
def find() = authenticated(Role.read).async(fieldsBodyParser) { implicit request
val query = request.body.getValue("query").fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
val range = request.body.getString("range")
val sort = request.body.getStrings("sort").getOrElse(Nil)
val nparent = request.body.getLong("nparent").getOrElse(0L).toInt
val withStats = request.body.getBoolean("nstats").getOrElse(false)

val (alerts, total) = alertSrv.find(query, range, sort)
val alertsWithStats = auxSrv.apply(alerts, nparent, withStats, removeUnaudited = false)
renderer.toOutput(OK, alertsWithStats, total)
}

@Timed
def stats() = authenticated(Role.read).async(fieldsBodyParser) { implicit request
val query = request.body.getValue("query")
.fold[QueryDef](QueryDSL.any)(_.as[QueryDef])
val aggs = request.body.getValue("stats")
.getOrElse(throw BadRequestError("Parameter \"stats\" is missing")).as[Seq[Agg]]
alertSrv.stats(query, aggs).map(s Ok(s))
}
}
83 changes: 83 additions & 0 deletions thehive-backend/app/models/Alert.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package models

import javax.inject.{ Inject, Singleton }

import models.JsonFormat.alertStatusFormat
import org.elastic4play.models.{ Attribute, AttributeDef, BaseEntity, EntityDef, HiveEnumeration, ModelDef }
import org.elastic4play.models.{ AttributeFormat F, AttributeOption O }
import play.api.Logger
import play.api.libs.json.{ JsObject, JsString, Json }
import services.AuditedModel

import scala.concurrent.Future

object AlertStatus extends Enumeration with HiveEnumeration {
type Type = Value
val New, Update, Ignore, Imported = Value
}

trait AlertAttributes {
_: AttributeDef
def artifactAttributes: Seq[Attribute[_]]

val alertId = attribute("_id", F.stringFmt, "Alert id", O.readonly)
val tpe = attribute("type", F.stringFmt, "Type of the alert", O.readonly)
val source = attribute("source", F.stringFmt, "Source of the alert", O.readonly)
val sourceRef = attribute("sourceRef", F.stringFmt, "Source reference of the alert", O.readonly)
val date = attribute("date", F.dateFmt, "Date of the alert", O.readonly)
val lastSyncDate = attribute("lastSyncDate", F.dateFmt, "Date of the last synchronization")
val caze = optionalAttribute("case", F.stringFmt, "Id of the case, if created")
val title = attribute("title", F.textFmt, "Title of the alert")
val description = attribute("description", F.textFmt, "Description of the alert")
val severity = attribute("severity", F.numberFmt, "Severity if the alert (0-5)", 3L)
val tags = multiAttribute("tags", F.stringFmt, "Alert tags")
val tlp = attribute("tlp", F.numberFmt, "TLP level", 2L)
val artifacts = multiAttribute("artifacts", F.objectFmt(artifactAttributes), "Artifact of the alert")
val caseTemplate = optionalAttribute("caseTemplate", F.stringFmt, "Case template to use")
val status = attribute("status", F.enumFmt(AlertStatus), "Status of the alert", AlertStatus.New)
val follow = attribute("follow", F.booleanFmt, "", true)
}

@Singleton
class AlertModel @Inject() (artifactModel: ArtifactModel)
extends ModelDef[AlertModel, Alert]("alert")
with AlertAttributes
with AuditedModel {

private[AlertModel] lazy val logger = Logger(getClass)
override val defaultSortBy: Seq[String] = Seq("-date")
override val removeAttribute: JsObject = Json.obj("status" AlertStatus.Ignore)

override def artifactAttributes: Seq[Attribute[_]] = artifactModel.attributes

override def creationHook(parent: Option[BaseEntity], attrs: JsObject): Future[JsObject] = {
Future.successful {
if (attrs.keys.contains("_id"))
attrs
else {
val tpe = (attrs \ "tpe").asOpt[String].getOrElse("<null>")
val source = (attrs \ "source").asOpt[String].getOrElse("<null>")
val sourceRef = (attrs \ "sourceRef").asOpt[String].getOrElse("<null>")
attrs + ("_id" JsString(s"$tpe|$source|$sourceRef"))
}
}
}
}

class Alert(model: AlertModel, attributes: JsObject)
extends EntityDef[AlertModel, Alert](model, attributes)
with AlertAttributes {

override def artifactAttributes: Seq[Attribute[_]] = Nil

def toCaseJson: JsObject = Json.obj(
//"caseId" -> caseId,
"title" title(),
"description" description(),
"severity" severity(),
//"owner" -> owner,
"startDate" date(),
"tags" tags(),
"tlp" tlp(),
"status" CaseStatus.Open)
}
1 change: 1 addition & 0 deletions thehive-backend/app/models/JsonFormat.scala
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ object JsonFormat {
implicit val taskStatusFormat = enumFormat(TaskStatus)
implicit val logStatusFormat = enumFormat(LogStatus)
implicit val caseTemplateStatusFormat = enumFormat(CaseTemplateStatus)
implicit val alertStatusFormat = enumFormat(AlertStatus)

implicit val pathWrites: Writes[Path] = Writes((value: Path) JsString(value.toString))
}
96 changes: 76 additions & 20 deletions thehive-backend/app/models/Migration.scala
Original file line number Diff line number Diff line change
@@ -1,48 +1,64 @@
package models

import java.util.Date

import javax.inject.Inject

import akka.stream.Materializer
import org.elastic4play.models.BaseModelDef
import org.elastic4play.services._
import org.elastic4play.utils
import org.elastic4play.utils.RichJson
import play.api.{ Configuration, Logger }
import play.api.libs.json.JsValue.jsValueToJsLookup
import play.api.libs.json._

import scala.collection.immutable.{ Set ISet }
import scala.concurrent.{ ExecutionContext, Future }
import scala.math.BigDecimal.int2bigDecimal
import scala.util.Try

import akka.stream.Materializer

import play.api.Logger
import play.api.libs.json.{ JsBoolean, JsNumber, JsObject, JsString, JsValue }
import play.api.libs.json.{ Json, Reads }
import play.api.libs.json.JsValue.jsValueToJsLookup

import org.elastic4play.models.BaseModelDef
import org.elastic4play.services.{ DBLists, DatabaseState, MigrationOperations, Operation }
import org.elastic4play.utils
import org.elastic4play.utils.RichJson
case class UpdateMispAlertArtifact() extends EventMessage

class Migration @Inject() (
class Migration(
mispCaseTemplate: Option[String],
models: ISet[BaseModelDef],
dblists: DBLists,
eventSrv: EventSrv,
implicit val ec: ExecutionContext,
implicit val materializer: Materializer) extends MigrationOperations {
@Inject() def this(
configuration: Configuration,
models: ISet[BaseModelDef],
dblists: DBLists,
eventSrv: EventSrv,
ec: ExecutionContext,
materializer: Materializer) = {
this(configuration.getString("misp.caseTemplate"), models, dblists, eventSrv, ec, materializer)
}

import org.elastic4play.services.Operation._
val logger = Logger(getClass)
private var requireUpdateMispAlertArtifact = false

override def beginMigration(version: Int) = Future.successful(())
override def beginMigration(version: Int): Future[Unit] = Future.successful(())

override def endMigration(version: Int) = {
log.info("Updating observable data type list")
override def endMigration(version: Int): Future[Unit] = {
if (requireUpdateMispAlertArtifact) {
logger.info("Retrieve MISP attribute to update alerts")
eventSrv.publish(UpdateMispAlertArtifact())
}
logger.info("Updating observable data type list")
val dataTypes = dblists.apply("list_artifactDataType")
Future.sequence(Seq("filename", "fqdn", "url", "user-agent", "domain", "ip", "mail_subject", "hash", "mail", "registry", "uri_path", "regexp", "other", "file")
Future.sequence(Seq("filename", "fqdn", "url", "user-agent", "domain", "ip", "mail_subject", "hash", "mail",
"registry", "uri_path", "regexp", "other", "file")
.map(dt dataTypes.addItem(dt).recover { case _ () }))
.map(_ ())
.recover { case _ () }
}

override val operations: PartialFunction[DatabaseState, Seq[Operation]] = {
case DatabaseState(version) if version < 7 Nil
case previousState @ DatabaseState(7)
case DatabaseState(7)
Seq(
renameAttribute("reportTemplate", "analyzerId", "analyzers"), // reportTemplate refers only one analyzer
renameAttribute("reportTemplate", "reportType", "flavor"), // rename flavor into reportType
Expand All @@ -69,10 +85,47 @@ class Migration @Inject() (
mapAttribute("misp", "publishDate")(convertDate),
mapAttribute(_ true, "createdAt", convertDate),
mapAttribute(_ true, "updatedAt", convertDate))
case DatabaseState(8)
requireUpdateMispAlertArtifact = true
Seq(
renameEntity("misp", "alert"),
mapEntity("alert") { misp
val eventId = (misp \ "eventId").as[Long].toString
val date = (misp \ "date").as[Date]
val mispTags = (misp \ "tags").asOpt[Seq[String]].getOrElse(Nil)
val tags = mispTags.filterNot(_.toLowerCase.startsWith("tlp:")) :+ s"src:${(misp \ "org").as[String]}"
val tlp = mispTags
.map(_.toLowerCase)
.collectFirst {
case "tlp:white" 0L
case "tlp:green" 1L
case "tlp:amber" 2L
case "tlp:red" 3L
}
.getOrElse(2L)
(misp \ "caze").asOpt[JsString].fold(JsObject(Nil))(c Json.obj("caze" c)) ++
Json.obj(
"_type" "alert",
"_id" ("misp:" + (misp \ "_id").as[String]),
"type" "misp",
"source" (misp \ "serverId").as[JsString],
"sourceRef" eventId,
"date" date,
"lastSyncDate" (misp \ "publishDate").as[Date],
"title" ("#" + eventId + " " + (misp \ "info").as[String]).trim,
"description" s"Imported from MISP Event #$eventId, created at $date",
"severity" (misp \ "threatLevel").as[JsNumber],
"tags" tags,
"tlp" tlp,
"artifacts" JsArray(),
"caseTemplate" mispCaseTemplate,
"status" (misp \ "eventStatus").as[JsString],
"follow" (misp \ "follow").as[JsBoolean])
})
}

private val requestCounter = new java.util.concurrent.atomic.AtomicInteger(0)
def getRequestId = {
def getRequestId: String = {
utils.Instance.id + ":mig:" + requestCounter.incrementAndGet()
}

Expand Down Expand Up @@ -113,7 +166,10 @@ class Migration @Inject() (
})
.map { attributes
// put audited attribute in details and unaudited in otherDetails
val otherDetails = (audit \ "otherDetails").asOpt[String].flatMap(od Try(Json.parse(od).as[JsObject]).toOption).getOrElse(JsObject(Nil))
val otherDetails = (audit \ "otherDetails")
.asOpt[String]
.flatMap(od Try(Json.parse(od).as[JsObject]).toOption)
.getOrElse(JsObject(Nil))
val (in, notIn) = details.fields.partition(f attributes.contains(f._1.split("\\.").head))
val newOtherDetails = otherDetails ++ JsObject(notIn)
audit + ("details" JsObject(in)) + ("otherDetails" JsString(newOtherDetails.toString.take(10000)))
Expand Down
2 changes: 1 addition & 1 deletion thehive-backend/app/models/package.scala
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@


package object models {
val version = 8
val version = 9
}
Loading

0 comments on commit 63adafe

Please sign in to comment.