Skip to content

Commit

Permalink
#1469 Add new filters on alerts and audits for migration
Browse files Browse the repository at this point in the history
  • Loading branch information
@To-om
To-om committed Nov 10, 2020
1 parent 28c3b78 commit 6e44c0c
Show file tree
Hide file tree
Showing 6 changed files with 171 additions and 80 deletions.
2 changes: 1 addition & 1 deletion ScalliGraph
Submodule ScalliGraph updated 2 files
+1 −1 core/src/main/scala/org/thp/scalligraph/auth/MultiAuthSrv.scala
+2 −1 core/src/main/scala/org/thp/scalligraph/traversal/TraversalOps.scala
8 changes: 8 additions & 0 deletions migration/src/main/resources/reference.conf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,14 @@ input {
maxCaseAge: 0
maxAlertAge: 0
maxAuditAge: 0
includeAlertTypes: []
excludeAlertTypes: []
includeAlertSources: []
excludeAlertSources: []
includeAuditActions: []
excludeAuditActions: []
includeAuditObjectTypes: []
excludeAuditObjectTypes: []
}

# Datastore
Expand Down
26 changes: 24 additions & 2 deletions migration/src/main/scala/org/thp/thehive/migration/Input.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,23 @@ import akka.stream.scaladsl.Source
import com.typesafe.config.Config
import org.thp.thehive.migration.dto._

import scala.collection.JavaConverters._
import scala.concurrent.Future
import scala.util.{Failure, Try}

case class Filter(
caseDateRange: (Option[Long], Option[Long]),
caseNumberRange: (Option[Int], Option[Int]),
alertDateRange: (Option[Long], Option[Long]),
auditDateRange: (Option[Long], Option[Long])
auditDateRange: (Option[Long], Option[Long]),
includeAlertTypes: Seq[String],
excludeAlertTypes: Seq[String],
includeAlertSources: Seq[String],
excludeAlertSources: Seq[String],
includeAuditActions: Seq[String],
excludeAuditActions: Seq[String],
includeAuditObjectTypes: Seq[String],
excludeAuditObjectTypes: Seq[String]
)

object Filter {
Expand Down Expand Up @@ -59,7 +68,20 @@ object Filter {
val auditFromDate = readDate("auditFromDate", "maxAuditAge")
val auditUntilDate = readDate("auditUntilDate", "minAuditAge")

Filter(caseFromDate -> caseUntilDate, caseFromNumber -> caseUntilNumber, alertFromDate -> alertUntilDate, auditFromDate -> auditUntilDate)
Filter(
caseFromDate -> caseUntilDate,
caseFromNumber -> caseUntilNumber,
alertFromDate -> alertUntilDate,
auditFromDate -> auditUntilDate,
config.getStringList("includeAlertTypes").asScala,
config.getStringList("excludeAlertTypes").asScala,
config.getStringList("includeAlertSources").asScala,
config.getStringList("excludeAlertSources").asScala,
config.getStringList("includeAuditActions").asScala,
config.getStringList("excludeAuditActions").asScala,
config.getStringList("includeAuditObjectTypes").asScala,
config.getStringList("excludeAuditObjectTypes").asScala
)
}
}

Expand Down
152 changes: 92 additions & 60 deletions migration/src/main/scala/org/thp/thehive/migration/Migrate.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ import java.io.File

import akka.actor.ActorSystem
import akka.stream.Materializer
import com.typesafe.config.{Config, ConfigFactory}
import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory}
import play.api.libs.logback.LogbackLoggerConfigurator
import play.api.{Configuration, Environment}
import scopt.OParser
Expand All @@ -17,9 +17,11 @@ object Migrate extends App with MigrationOps {
Option(System.getProperty("logger.file")).getOrElse {
System.setProperty("logger.file", "/etc/thehive/logback-migration.xml")
}

def getVersion: String = Option(getClass.getPackage.getImplementationVersion).getOrElse("SNAPSHOT")
def addConfig(config: Config, settings: (String, Any)*): Config =
ConfigFactory.parseMap(Map(settings: _*).asJava).withFallback(config)

def addConfig(config: Config, path: String, value: Any): Config =
config.withValue(path, ConfigValueFactory.fromAnyRef(value))

val builder = OParser.builder[Config]
val argParser = {
Expand All @@ -35,93 +37,121 @@ object Migrate extends App with MigrationOps {
.text("global configuration file"),
opt[File]('i', "input")
.valueName("<file>")
.action((f, c) => addConfig(c, "input" -> ConfigFactory.parseFileAnySyntax(f).resolve().root()))
.action((f, c) => addConfig(c, "input", ConfigFactory.parseFileAnySyntax(f).resolve().root()))
.text("TheHive3 configuration file"),
opt[File]('o', "output")
.valueName("<file>")
.action((f, c) => addConfig(c, "output" -> ConfigFactory.parseFileAnySyntax(f).resolve().root()))
.action((f, c) => addConfig(c, "output", ConfigFactory.parseFileAnySyntax(f).resolve().root()))
.text("TheHive4 configuration file"),
opt[Unit]('d', "drop-database")
.action((_, c) => addConfig(c, "output.dropDatabase" -> true))
.action((_, c) => addConfig(c, "output.dropDatabase", true))
.text("Drop TheHive4 database before migration"),
opt[String]('m', "main-organisation")
.valueName("<organisation>")
.action((o, c) => addConfig(c, "input.mainOrganisation" -> o)),
.action((o, c) => addConfig(c, "input.mainOrganisation", o)),
opt[String]('u', "es-uri")
.valueName("http://ip1:port,ip2:port")
.text("TheHive3 ElasticSearch URI")
.action((u, c) => addConfig(c, "input.search.uri" -> u)),
.action((u, c) => addConfig(c, "input.search.uri", u)),
opt[String]('i', "es-index")
.valueName("<index>")
.text("TheHive3 ElasticSearch index name")
.action((i, c) => addConfig(c, "intput.search.index" -> i)),
opt[Duration]('a', "es-keepalive")
.action((i, c) => addConfig(c, "intput.search.index", i)),
opt[String]('a', "es-keepalive")
.valueName("<duration>")
.text("TheHive3 ElasticSearch keepalive")
.action((a, c) => addConfig(c, "input.search.keepalive" -> a.toString)),
.action((a, c) => addConfig(c, "input.search.keepalive", a)),
opt[Int]('p', "es-pagesize")
.text("TheHive3 ElasticSearch page size")
.action((p, c) => addConfig(c, "input.search.pagesize" -> p)),
.action((p, c) => addConfig(c, "input.search.pagesize", p)),
/* case age */
opt[Duration]("max-case-age")
opt[String]("max-case-age")
.valueName("<duration>")
.text("migrate only cases whose age is less than <duration>")
.action((v, c) => addConfig(c, "input.filter.maxCaseAge" -> v.toString)),
opt[Duration]("min-case-age")
.action((v, c) => addConfig(c, "input.filter.maxCaseAge", v)),
opt[String]("min-case-age")
.valueName("<duration>")
.text("migrate only cases whose age is greater than <duration>")
.action((v, c) => addConfig(c, "input.filter.minCaseAge" -> v.toString)),
.action((v, c) => addConfig(c, "input.filter.minCaseAge", v)),
opt[String]("case-from-date")
.valueName("<date>")
.text("migrate only cases created from <date>")
.action((v, c) => addConfig(c, "input.filter.caseFromDate" -> v)),
.action((v, c) => addConfig(c, "input.filter.caseFromDate", v)),
opt[String]("case-until-date")
.valueName("<date>")
.text("migrate only cases created until <date>")
.action((v, c) => addConfig(c, "input.filter.caseUntilDate" -> v)),
.action((v, c) => addConfig(c, "input.filter.caseUntilDate", v)),
/* case number */
opt[Duration]("case-from-number")
opt[Int]("case-from-number")
.valueName("<number>")
.text("migrate only cases from this case number")
.action((v, c) => addConfig(c, "input.filter.caseFromNumber" -> v.toString)),
opt[Duration]("case-until-number")
.action((v, c) => addConfig(c, "input.filter.caseFromNumber", v)),
opt[Int]("case-until-number")
.valueName("<number>")
.text("migrate only cases until this case number")
.action((v, c) => addConfig(c, "input.filter.caseUntilNumber" -> v.toString)),
.action((v, c) => addConfig(c, "input.filter.caseUntilNumber", v)),
/* alert age */
opt[Duration]("max-alert-age")
opt[String]("max-alert-age")
.valueName("<duration>")
.text("migrate only alerts whose age is less than <duration>")
.action((v, c) => addConfig(c, "input.filter.maxAlertAge" -> v.toString)),
opt[Duration]("min-alert-age")
.action((v, c) => addConfig(c, "input.filter.maxAlertAge", v)),
opt[String]("min-alert-age")
.valueName("<duration>")
.text("migrate only alerts whose age is greater than <duration>")
.action((v, c) => addConfig(c, "input.filter.minAlertAge" -> v.toString)),
.action((v, c) => addConfig(c, "input.filter.minAlertAge", v)),
opt[String]("alert-from-date")
.valueName("<date>")
.text("migrate only alerts created from <date>")
.action((v, c) => addConfig(c, "input.filter.alertFromDate" -> v)),
.action((v, c) => addConfig(c, "input.filter.alertFromDate", v)),
opt[String]("alert-until-date")
.valueName("<date>")
.text("migrate only alerts created until <date>")
.action((v, c) => addConfig(c, "input.filter.alertUntilDate" -> v)),
.action((v, c) => addConfig(c, "input.filter.alertUntilDate", v)),
opt[Seq[String]]("include-alert-types")
.valueName("<type>,<type>...")
.text("migrate only alerts with this types")
.action((v, c) => addConfig(c, "input.filter.includeAlertTypes", v.asJava)),
opt[Seq[String]]("exclude-alert-types")
.valueName("<type>,<type>...")
.text("don't migrate alerts with this types")
.action((v, c) => addConfig(c, "input.filter.excludeAlertTypes", v.asJava)),
opt[Seq[String]]("include-alert-sources")
.valueName("<source>,<source>...")
.text("migrate only alerts with this sources")
.action((v, c) => addConfig(c, "input.filter.includeAlertSources", v.asJava)),
opt[Seq[String]]("exclude-alert-sources")
.valueName("<source>,<source>...")
.text("don't migrate alerts with this sources")
.action((v, c) => addConfig(c, "input.filter.excludeAlertSources", v.asJava)),
/* audit age */
opt[Duration]("max-audit-age")
opt[String]("max-audit-age")
.valueName("<duration>")
.text("migrate only audits whose age is less than <duration>")
.action((v, c) => addConfig(c, "input.filter.minAuditAge" -> v.toString)),
opt[Duration]("min-audit-age")
.action((v, c) => addConfig(c, "input.filter.minAuditAge", v)),
opt[String]("min-audit-age")
.valueName("<duration>")
.text("migrate only audits whose age is greater than <duration>")
.action((v, c) => addConfig(c, "input.filter.maxAuditAge" -> v.toString)),
.action((v, c) => addConfig(c, "input.filter.maxAuditAge", v)),
opt[String]("audit-from-date")
.valueName("<date>")
.text("migrate only audits created from <date>")
.action((v, c) => addConfig(c, "input.filter.auditFromDate" -> v)),
.action((v, c) => addConfig(c, "input.filter.auditFromDate", v)),
opt[String]("audit-until-date")
.valueName("<date>")
.text("migrate only audits created until <date>")
.action((v, c) => addConfig(c, "input.filter.auditUntilDate" -> v)),
.action((v, c) => addConfig(c, "input.filter.auditUntilDate", v)),
opt[Seq[String]]("include-audit-actions")
.text("migration only audits with this action (Update, Creation, Delete)")
.action((v, c) => addConfig(c, "input.filter.includeAuditActions", v.asJava)),
opt[Seq[String]]("exclude-audit-actions")
.text("don't migration audits with this action (Update, Creation, Delete)")
.action((v, c) => addConfig(c, "input.filter.excludeAuditActions", v.asJava)),
opt[Seq[String]]("include-audit-objectTypes")
.text("migration only audits with this objectType (case, case_artifact, case_task, ...)")
.action((v, c) => addConfig(c, "input.filter.includeAuditObjectTypes", v.asJava)),
opt[Seq[String]]("exclude-audit-objectTypes")
.text("don't migration audits with this objectType (case, case_artifact, case_task, ...)")
.action((v, c) => addConfig(c, "input.filter.excludeAuditObjectTypes", v.asJava)),
note("Accepted date formats are \"yyyyMMdd[HH[mm[ss]]]\" and \"MMdd\""),
note(
"The Format for duration is: <length> <unit>.\n" +
Expand All @@ -144,35 +174,37 @@ object Migrate extends App with MigrationOps {
implicit val ec: ExecutionContext = actorSystem.dispatcher
implicit val mat: Materializer = Materializer(actorSystem)

(new LogbackLoggerConfigurator).configure(Environment.simple(), Configuration.empty, Map.empty)
try {
(new LogbackLoggerConfigurator).configure(Environment.simple(), Configuration.empty, Map.empty)

val timer = actorSystem.scheduler.scheduleAtFixedRate(10.seconds, 10.seconds) { () =>
logger.info(migrationStats.showStats())
migrationStats.flush()
}
val timer = actorSystem.scheduler.scheduleAtFixedRate(10.seconds, 10.seconds) { () =>
logger.info(migrationStats.showStats())
migrationStats.flush()
}

val returnStatus =
try {
val input = th3.Input(Configuration(config.getConfig("input").withFallback(config)))
val output = th4.Output(Configuration(config.getConfig("output").withFallback(config)))
val filter = Filter.fromConfig(config.getConfig("input.filter"))
val returnStatus =
try {
val input = th3.Input(Configuration(config.getConfig("input").withFallback(config)))
val output = th4.Output(Configuration(config.getConfig("output").withFallback(config)))
val filter = Filter.fromConfig(config.getConfig("input.filter"))

val process = migrate(input, output, filter)
val process = migrate(input, output, filter)

Await.result(process, Duration.Inf)
logger.info("Migration finished")
0
} catch {
case e: Throwable =>
logger.error(s"Migration failed", e)
1
} finally {
timer.cancel()
Await.ready(actorSystem.terminate(), 1.minute)
()
}
migrationStats.flush()
logger.info(migrationStats.toString)
System.exit(returnStatus)
Await.result(process, Duration.Inf)
logger.info("Migration finished")
0
} catch {
case e: Throwable =>
logger.error(s"Migration failed", e)
1
} finally {
timer.cancel()
Await.ready(actorSystem.terminate(), 1.minute)
()
}
migrationStats.flush()
logger.info(migrationStats.toString)
System.exit(returnStatus)
} finally actorSystem.terminate()
}
}
41 changes: 36 additions & 5 deletions migration/src/main/scala/org/thp/thehive/migration/th3/Input.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,9 @@ import akka.stream.Materializer
import akka.stream.scaladsl.Source
import akka.util.ByteString
import com.google.inject.Guice
import com.sksamuel.elastic4s.http.ElasticDsl.{bool, hasParentQuery, idsQuery, rangeQuery, search, termQuery}
import com.sksamuel.elastic4s.http.ElasticDsl._
import com.sksamuel.elastic4s.searches.queries.RangeQuery
import com.sksamuel.elastic4s.searches.queries.term.TermsQuery
import javax.inject.{Inject, Singleton}
import net.codingwell.scalaguice.ScalaModule
import org.thp.thehive.migration
Expand Down Expand Up @@ -304,14 +305,28 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe
Seq(fromFilter.andThen(untilFilter).apply(rangeQuery("createdAt")))
} else Nil

def alertIncludeFilter(filter: Filter): Seq[TermsQuery[String]] =
(if (filter.includeAlertTypes.nonEmpty) Seq(termsQuery("type", filter.includeAlertTypes)) else Nil) ++
(if (filter.includeAlertSources.nonEmpty) Seq(termsQuery("source", filter.includeAlertSources)) else Nil)

def alertExcludeFilter(filter: Filter): Seq[TermsQuery[String]] =
(if (filter.excludeAlertTypes.nonEmpty) Seq(termsQuery("type", filter.excludeAlertTypes)) else Nil) ++
(if (filter.excludeAlertSources.nonEmpty) Seq(termsQuery("source", filter.excludeAlertSources)) else Nil)

override def listAlerts(filter: Filter): Source[Try[InputAlert], NotUsed] =
dbFind(Some("all"), Seq("-createdAt"))(indexName =>
search(indexName).query(bool(alertFilter(filter) :+ termQuery("relations", "alert"), Nil, Nil))
search(indexName).query(
bool((alertFilter(filter) :+ termQuery("relations", "alert")) ++ alertIncludeFilter(filter), Nil, alertExcludeFilter(filter))
)
)._1
.read[InputAlert]

override def countAlerts(filter: Filter): Future[Long] =
dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(bool(alertFilter(filter) :+ termQuery("relations", "alert"), Nil, Nil)))._2
dbFind(Some("0-0"), Nil)(indexName =>
search(indexName).query(
bool((alertFilter(filter) :+ termQuery("relations", "alert")) ++ alertIncludeFilter(filter), Nil, alertExcludeFilter(filter))
)
)._2

override def listAlertObservables(filter: Filter): Source[Try[(String, InputObservable)], NotUsed] =
dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(alertFilter(filter) :+ termQuery("relations", "alert"), Nil, Nil)))
Expand Down Expand Up @@ -638,13 +653,29 @@ class Input @Inject() (configuration: Configuration, dbFind: DBFind, dbGet: DBGe
Seq(fromFilter.andThen(untilFilter).apply(rangeQuery("createdAt")))
} else Nil

def auditIncludeFilter(filter: Filter): Seq[TermsQuery[String]] =
(if (filter.includeAuditActions.nonEmpty) Seq(termsQuery("operation", filter.includeAuditActions)) else Nil) ++
(if (filter.includeAuditObjectTypes.nonEmpty) Seq(termsQuery("objectType", filter.includeAuditObjectTypes)) else Nil)

def auditExcludeFilter(filter: Filter): Seq[TermsQuery[String]] =
(if (filter.excludeAuditActions.nonEmpty) Seq(termsQuery("operation", filter.excludeAuditActions)) else Nil) ++
(if (filter.excludeAuditObjectTypes.nonEmpty) Seq(termsQuery("objectType", filter.excludeAuditObjectTypes)) else Nil)

override def listAudit(filter: Filter): Source[Try[(String, InputAudit)], NotUsed] =
dbFind(Some("all"), Nil)(indexName => search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit"), Nil, Nil)))
dbFind(Some("all"), Nil)(indexName =>
search(indexName).query(
bool((auditFilter(filter) :+ termQuery("relations", "audit")) ++ auditIncludeFilter(filter), Nil, auditExcludeFilter(filter))
)
)
._1
.read[(String, InputAudit)]

override def countAudit(filter: Filter): Future[Long] =
dbFind(Some("0-0"), Nil)(indexName => search(indexName).query(bool(auditFilter(filter) :+ termQuery("relations", "audit"), Nil, Nil)))._2
dbFind(Some("0-0"), Nil)(indexName =>
search(indexName).query(
bool((auditFilter(filter) :+ termQuery("relations", "audit")) ++ auditIncludeFilter(filter), Nil, auditExcludeFilter(filter))
)
)._2

override def listAudit(entityId: String, filter: Filter): Source[Try[(String, InputAudit)], NotUsed] =
dbFind(Some("all"), Nil)(indexName =>
Expand Down
Loading

0 comments on commit 6e44c0c

Please sign in to comment.