Skip to content

Commit

Permalink
#2024 Add constraints on input data
Browse files Browse the repository at this point in the history
  • Loading branch information
To-om committed May 21, 2021
1 parent 4ca7b0d commit 140539c
Show file tree
Hide file tree
Showing 61 changed files with 477 additions and 283 deletions.
2 changes: 1 addition & 1 deletion ScalliGraph
Submodule ScalliGraph updated 43 files
+2 −1 build.sbt
+0 −135 core-test/src/test/scala/org/thp/scalligraph/AppBuilder.scala
+4 −5 core-test/src/test/scala/org/thp/scalligraph/RetryTest.scala
+0 −14 core-test/src/test/scala/org/thp/scalligraph/controllers/ControllerTest.scala
+1 −1 core-test/src/test/scala/org/thp/scalligraph/models/CallbackTest.scala
+3 −10 core-test/src/test/scala/org/thp/scalligraph/models/CardinalityTest.scala
+1 −3 core-test/src/test/scala/org/thp/scalligraph/models/DatabaseProviders.scala
+1 −1 core-test/src/test/scala/org/thp/scalligraph/models/IndexTest.scala
+25 −28 core-test/src/test/scala/org/thp/scalligraph/models/Modern.scala
+1 −3 core-test/src/test/scala/org/thp/scalligraph/models/ModernQuery.scala
+41 −44 core-test/src/test/scala/org/thp/scalligraph/models/ModernTest.scala
+102 −107 core-test/src/test/scala/org/thp/scalligraph/models/QueryTest.scala
+4 −4 core-test/src/test/scala/org/thp/scalligraph/models/SimpleEntityTest.scala
+60 −64 core-test/src/test/scala/org/thp/scalligraph/services/IntegrityCheckTest.scala
+3 −3 core-test/src/test/scala/org/thp/scalligraph/services/StorageSrvTest.scala
+2 −0 core/src/main/resources/reference.conf
+6 −0 core/src/main/scala/org/thp/scalligraph/EntityId.scala
+131 −90 core/src/main/scala/org/thp/scalligraph/ScalligraphApplication.scala
+3 −4 core/src/main/scala/org/thp/scalligraph/auth/MultiAuthSrv.scala
+19 −1 core/src/main/scala/org/thp/scalligraph/controllers/FieldsParser.scala
+2 −2 core/src/main/scala/org/thp/scalligraph/models/Database.scala
+7 −4 core/src/main/scala/org/thp/scalligraph/models/Mapping.scala
+3 −4 core/src/main/scala/org/thp/scalligraph/models/Model.scala
+2 −2 core/src/main/scala/org/thp/scalligraph/models/Operation.scala
+4 −1 core/src/main/scala/org/thp/scalligraph/models/Schema.scala
+4 −4 core/src/main/scala/org/thp/scalligraph/query/Aggregation.scala
+5 −6 core/src/main/scala/org/thp/scalligraph/query/Filter.scala
+1 −1 core/src/main/scala/org/thp/scalligraph/query/PredicateOps.scala
+6 −5 core/src/main/scala/org/thp/scalligraph/query/PublicProperty.scala
+3 −4 core/src/main/scala/org/thp/scalligraph/query/Query.scala
+0 −1 core/src/main/scala/org/thp/scalligraph/services/EdgeSrv.scala
+2 −2 core/src/main/scala/org/thp/scalligraph/services/ElementSrv.scala
+2 −2 core/src/main/scala/org/thp/scalligraph/services/EventSrv.scala
+2 −2 core/src/main/scala/org/thp/scalligraph/services/IntegrityCheckOps.scala
+1 −3 core/src/main/scala/org/thp/scalligraph/services/StorageSrv.scala
+0 −1 core/src/main/scala/org/thp/scalligraph/services/VertexSrv.scala
+1 −2 core/src/main/scala/org/thp/scalligraph/traversal/IteratorOutput.scala
+2 −3 core/src/main/scala/org/thp/scalligraph/traversal/MatchElement.scala
+8 −8 core/src/main/scala/org/thp/scalligraph/traversal/TraversalOps.scala
+2 −2 core/src/main/scala/org/thp/scalligraph/utils/Retry.scala
+23 −14 database/janusgraph/src/main/scala/org/thp/scalligraph/janus/JanusDatabase.scala
+7 −7 database/janusgraph/src/main/scala/org/thp/scalligraph/janus/JanusDatabaseProvider.scala
+5 −1 project/Dependencies.scala
4 changes: 3 additions & 1 deletion build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,9 @@ lazy val thehiveDto = (project in file("dto"))
name := "thehive-dto",
version := thehiveVersion,
libraryDependencies ++= Seq(
aix
aix,
refined,
playRefined
)
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import org.thp.scalligraph.traversal.Graph
import org.thp.scalligraph.{EntityIdOrName, InternalError}
import org.thp.thehive.connector.cortex.models._
import org.thp.thehive.controllers.v0.Conversion._
import org.thp.thehive.dto.{Description, String128}
import org.thp.thehive.dto.v0.InputTask
import org.thp.thehive.models._
import org.thp.thehive.services._
Expand Down Expand Up @@ -58,7 +59,10 @@ class ActionOperationSrv(
case CreateTask(title, description) =>
for {
case0 <- relatedCase.fold[Try[Case with Entity]](Failure(InternalError("Unable to apply action CreateTask without case")))(Success(_))
_ <- caseSrv.createTask(case0, InputTask(title = title, description = Some(description)).toTask)
_ <- caseSrv.createTask(
case0,
InputTask(title = String128("task.title", title), description = Some(Description("task.description", description))).toTask
)
} yield updateOperation(operation)

case AddCustomFields(name, _, value) =>
Expand Down
59 changes: 59 additions & 0 deletions dto/src/main/scala/org/thp/thehive/dto/package.scala
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
package org.thp.thehive

import eu.timepit.refined.W
import eu.timepit.refined.api.{RefType, Refined}
import shapeless.{::, HNil}
import eu.timepit.refined.predicates.all._
import org.thp.scalligraph.InvalidFormatAttributeError
import org.thp.scalligraph.controllers.{FNumber, FString}

package object dto {
type Severity = Int Refined And[Positive, LessEqual[W.`4`.T]]
type Tlp = Int Refined And[NonNegative, LessEqual[W.`3`.T]]
type Pap = Int Refined And[NonNegative, LessEqual[W.`3`.T]]
type Color = String Refined MatchesRegex[W.`"""^#[0-9a-fA-F]{6,6}|$"""`.T]
type StringX[N] = String Refined AllOf[Not[Empty] :: MaxSize[N] :: Not[MatchesRegex[W.`"""[\n\r]"""`.T]] :: HNil]
type String16 = StringX[W.`16`.T]
type String32 = StringX[W.`32`.T]
type String64 = StringX[W.`64`.T]
type String128 = StringX[W.`128`.T]
type String512 = StringX[W.`512`.T]
type Description = String Refined MaxSize[W.`1048576`.T]

object Severity {
def apply(value: Int): Severity =
RefType
.applyRef[Severity](value)
.fold(error => throw InvalidFormatAttributeError("severity", error, Set.empty, FNumber(value.toDouble)), identity)
}

object Tlp {
def apply(value: Int): Tlp =
RefType.applyRef[Tlp](value).fold(error => throw InvalidFormatAttributeError("tlp", error, Set.empty, FNumber(value.toDouble)), identity)
}

object Pap {
def apply(value: Int): Pap =
RefType.applyRef[Pap](value).fold(error => throw InvalidFormatAttributeError("pap", error, Set.empty, FNumber(value.toDouble)), identity)
}

object String64 {
def apply(name: String, value: String): String64 =
RefType.applyRef[String64](value).fold(error => throw InvalidFormatAttributeError(name, error, Set.empty, FString(value)), identity)
}

object String128 {
def apply(name: String, value: String): String128 =
RefType.applyRef[String128](value).fold(error => throw InvalidFormatAttributeError(name, error, Set.empty, FString(value)), identity)
}

object String512 {
def apply(name: String, value: String): String512 =
RefType.applyRef[String512](value).fold(error => throw InvalidFormatAttributeError(name, error, Set.empty, FString(value)), identity)
}

object Description {
def apply(name: String, value: String): Description =
RefType.applyRef[Description](value).fold(error => throw InvalidFormatAttributeError(name, error, Set.empty, FString(value)), identity)
}
}
22 changes: 12 additions & 10 deletions dto/src/main/scala/org/thp/thehive/dto/v0/Alert.scala
Original file line number Diff line number Diff line change
Expand Up @@ -2,22 +2,24 @@ package org.thp.thehive.dto.v0

import org.thp.scalligraph.controllers.WithParser
import play.api.libs.json._
import be.venneborg.refined.play.RefinedJsonFormats._
import org.thp.thehive.dto.{Description, Pap, Severity, String128, String16, String512, Tlp}

import java.util.Date

case class InputAlert(
`type`: String,
source: String,
sourceRef: String,
externalLink: Option[String],
title: String,
description: String,
severity: Option[Int] = None,
`type`: String16,
source: String16,
sourceRef: String128,
externalLink: Option[String512],
title: String512,
description: Description,
severity: Option[Severity] = None,
date: Option[Date] = None,
tags: Set[String] = Set.empty,
tags: Set[String128] = Set.empty,
flag: Option[Boolean] = None,
tlp: Option[Int] = None,
pap: Option[Int] = None,
tlp: Option[Tlp] = None,
pap: Option[Pap] = None,
@WithParser(InputCustomFieldValue.parser)
customFields: Seq[InputCustomFieldValue] = Nil
)
Expand Down
4 changes: 3 additions & 1 deletion dto/src/main/scala/org/thp/thehive/dto/v0/Attachment.scala
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package org.thp.thehive.dto.v0

import org.thp.thehive.dto.String128
import play.api.libs.json.{Json, OFormat, Writes}
import be.venneborg.refined.play.RefinedJsonFormats._

case class InputAttachment(name: String, contentType: String, id: String)
case class InputAttachment(name: String128, contentType: String128, id: String128)

object InputAttachment {
implicit val writes: Writes[InputAttachment] = Json.writes[InputAttachment]
Expand Down
20 changes: 11 additions & 9 deletions dto/src/main/scala/org/thp/thehive/dto/v0/Case.scala
Original file line number Diff line number Diff line change
Expand Up @@ -4,20 +4,22 @@ import org.thp.scalligraph.controllers.WithParser
import play.api.libs.json._

import java.util.Date
import be.venneborg.refined.play.RefinedJsonFormats._
import org.thp.thehive.dto.{Description, Pap, Severity, String128, String16, String512, Tlp}

case class InputCase(
title: String,
description: String,
severity: Option[Int] = None,
title: String512,
description: Description,
severity: Option[Severity] = None,
startDate: Option[Date] = None,
endDate: Option[Date] = None,
tags: Set[String] = Set.empty,
tags: Set[String128] = Set.empty,
flag: Option[Boolean] = None,
tlp: Option[Int] = None,
pap: Option[Int] = None,
status: Option[String] = None,
summary: Option[String] = None,
user: Option[String] = None,
tlp: Option[Tlp] = None,
pap: Option[Pap] = None,
status: Option[String16] = None,
summary: Option[Description] = None,
user: Option[String128] = None,
@WithParser(InputCustomFieldValue.parser)
customFields: Seq[InputCustomFieldValue] = Nil
)
Expand Down
20 changes: 11 additions & 9 deletions dto/src/main/scala/org/thp/thehive/dto/v0/CaseTemplate.scala
Original file line number Diff line number Diff line change
@@ -1,21 +1,23 @@
package org.thp.thehive.dto.v0

import org.thp.scalligraph.controllers.WithParser
import org.thp.thehive.dto.{Description, Pap, Severity, String128, String64, Tlp}
import play.api.libs.json.{JsObject, Json, OFormat, OWrites}

import java.util.Date
import be.venneborg.refined.play.RefinedJsonFormats._

case class InputCaseTemplate(
name: String,
displayName: Option[String],
titlePrefix: Option[String],
description: Option[String],
severity: Option[Int] = None,
tags: Set[String] = Set.empty,
name: String64,
displayName: Option[String64],
titlePrefix: Option[String64],
description: Option[Description],
severity: Option[Severity] = None,
tags: Set[String128] = Set.empty,
flag: Option[Boolean] = None,
tlp: Option[Int] = None,
pap: Option[Int] = None,
summary: Option[String] = None,
tlp: Option[Tlp] = None,
pap: Option[Pap] = None,
summary: Option[Description] = None,
tasks: Seq[InputTask] = Nil,
@WithParser(InputCustomFieldValue.parser)
customFields: Seq[InputCustomFieldValue] = Nil
Expand Down
64 changes: 33 additions & 31 deletions dto/src/main/scala/org/thp/thehive/dto/v0/CustomFieldValue.scala
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ import org.scalactic.Accumulation._
import org.scalactic._
import org.thp.scalligraph.controllers.{FNull, _}
import org.thp.scalligraph.{AttributeError, InvalidFormatAttributeError}
import org.thp.thehive.dto.{Description, String16, String64}
import play.api.libs.json._

import java.util.Date
import be.venneborg.refined.play.RefinedJsonFormats._

case class OutputCustomField(
id: String,
Expand All @@ -22,43 +24,43 @@ object OutputCustomField {
implicit val format: OFormat[OutputCustomField] = Json.format[OutputCustomField]
}

case class InputCustomFieldValue(name: String, value: Option[Any], order: Option[Int])
case class InputCustomFieldValue(name: String64, value: Option[Any], order: Option[Int])

object InputCustomFieldValue {

def getStringCustomField(name: String, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
def getStringCustomField(name: String64, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
obj.get("string") match {
case FUndefined => None
case FNull => Some(Good(InputCustomFieldValue(name, None, obj.getNumber("order").map(_.toInt))))
case FString(value) => Some(Good(InputCustomFieldValue(name, Some(value), obj.getNumber("order").map(_.toInt))))
case other => Some(Bad(One(InvalidFormatAttributeError(s"customField.$name.string", "string", Set.empty, other))))
}

def getIntegerCustomField(name: String, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
def getIntegerCustomField(name: String64, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
obj.get("integer") match {
case FUndefined => None
case FNull => Some(Good(InputCustomFieldValue(name, None, obj.getNumber("order").map(_.toInt))))
case FNumber(value) => Some(Good(InputCustomFieldValue(name, Some(value.toInt), obj.getNumber("order").map(_.toInt))))
case other => Some(Bad(One(InvalidFormatAttributeError(s"customField.$name.integer", "integer", Set.empty, other))))
}

def getFloatCustomField(name: String, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
def getFloatCustomField(name: String64, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
obj.get("float") match {
case FUndefined => None
case FNull => Some(Good(InputCustomFieldValue(name, None, obj.getNumber("order").map(_.toInt))))
case FNumber(value) => Some(Good(InputCustomFieldValue(name, Some(value.toDouble), obj.getNumber("order").map(_.toInt))))
case FNumber(value) => Some(Good(InputCustomFieldValue(name, Some(value), obj.getNumber("order").map(_.toInt))))
case other => Some(Bad(One(InvalidFormatAttributeError(s"customField.$name.float", "float", Set.empty, other))))
}

def getDateCustomField(name: String, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
def getDateCustomField(name: String64, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
obj.get("date") match {
case FUndefined => None
case FNull => Some(Good(InputCustomFieldValue(name, None, obj.getNumber("order").map(_.toInt))))
case FNumber(value) => Some(Good(InputCustomFieldValue(name, Some(new Date(value.toLong)), obj.getNumber("order").map(_.toInt))))
case other => Some(Bad(One(InvalidFormatAttributeError(s"customField.$name.date", "date", Set.empty, other))))
}

def getBooleanCustomField(name: String, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
def getBooleanCustomField(name: String64, obj: FObject): Option[Or[InputCustomFieldValue, Every[AttributeError]]] =
obj.get("boolean") match {
case FUndefined => None
case FNull => Some(Good(InputCustomFieldValue(name, None, obj.getNumber("order").map(_.toInt))))
Expand All @@ -71,18 +73,18 @@ object InputCustomFieldValue {
fields
.toSeq
.validatedBy {
case (name, FString(value)) => Good(InputCustomFieldValue(name, Some(value), None))
case (name, FNumber(value)) => Good(InputCustomFieldValue(name, Some(value), None))
case (name, FBoolean(value)) => Good(InputCustomFieldValue(name, Some(value), None))
case (name, FAny(value :: _)) => Good(InputCustomFieldValue(name, Some(value), None))
case (name, FNull) => Good(InputCustomFieldValue(name, None, None))
case (name, FString(value)) => Good(InputCustomFieldValue(String64("customFieldValue.name", name), Some(value), None))
case (name, FNumber(value)) => Good(InputCustomFieldValue(String64("customFieldValue.name", name), Some(value), None))
case (name, FBoolean(value)) => Good(InputCustomFieldValue(String64("customFieldValue.name", name), Some(value), None))
case (name, FAny(value :: _)) => Good(InputCustomFieldValue(String64("customFieldValue.name", name), Some(value), None))
case (name, FNull) => Good(InputCustomFieldValue(String64("customFieldValue.name", name), None, None))
case (name, obj: FObject) =>
getStringCustomField(name, obj) orElse
getIntegerCustomField(name, obj) orElse
getFloatCustomField(name, obj) orElse
getDateCustomField(name, obj) orElse
getBooleanCustomField(name, obj) getOrElse
Good(InputCustomFieldValue(name, None, None))
getStringCustomField(String64("customFieldValue.name", name), obj) orElse
getIntegerCustomField(String64("customFieldValue.name", name), obj) orElse
getFloatCustomField(String64("customFieldValue.name", name), obj) orElse
getDateCustomField(String64("customFieldValue.name", name), obj) orElse
getBooleanCustomField(String64("customFieldValue.name", name), obj) getOrElse
Good(InputCustomFieldValue(String64("customFieldValue.name", name), None, None))
case (name, other) =>
Bad(
One(
Expand All @@ -95,26 +97,26 @@ object InputCustomFieldValue {
}

implicit val writes: Writes[Seq[InputCustomFieldValue]] = Writes[Seq[InputCustomFieldValue]] { icfv =>
val fields = icfv.map {
case InputCustomFieldValue(name, Some(s: String), _) => name -> JsString(s)
case InputCustomFieldValue(name, Some(l: Long), _) => name -> JsNumber(l)
case InputCustomFieldValue(name, Some(d: Double), _) => name -> JsNumber(d)
case InputCustomFieldValue(name, Some(i: Integer), _) => name -> JsNumber(i.toLong)
case InputCustomFieldValue(name, Some(f: Float), _) => name -> JsNumber(f.toDouble)
case InputCustomFieldValue(name, Some(b: Boolean), _) => name -> JsBoolean(b)
case InputCustomFieldValue(name, Some(d: Date), _) => name -> JsNumber(d.getTime)
case InputCustomFieldValue(name, None, _) => name -> JsNull
val fields: Seq[(String, JsValue)] = icfv.map {
case InputCustomFieldValue(name, Some(s: String), _) => name.value -> JsString(s)
case InputCustomFieldValue(name, Some(l: Long), _) => name.value -> JsNumber(l)
case InputCustomFieldValue(name, Some(d: Double), _) => name.value -> JsNumber(d)
case InputCustomFieldValue(name, Some(i: Integer), _) => name.value -> JsNumber(i.toLong)
case InputCustomFieldValue(name, Some(f: Float), _) => name.value -> JsNumber(f.toDouble)
case InputCustomFieldValue(name, Some(b: Boolean), _) => name.value -> JsBoolean(b)
case InputCustomFieldValue(name, Some(d: Date), _) => name.value -> JsNumber(d.getTime)
case InputCustomFieldValue(name, None, _) => name.value -> JsNull
case InputCustomFieldValue(name, other, _) => sys.error(s"The custom field $name has invalid value: $other (${other.getClass})")
}
JsObject(fields)
}
}

case class InputCustomField(
name: String,
description: String,
`type`: String,
reference: String,
name: String64,
description: Description,
`type`: String16,
reference: String64,
mandatory: Option[Boolean],
options: Seq[JsValue] = Nil
)
Expand Down
9 changes: 8 additions & 1 deletion dto/src/main/scala/org/thp/thehive/dto/v0/Dashboard.scala
Original file line number Diff line number Diff line change
@@ -1,10 +1,17 @@
package org.thp.thehive.dto.v0

import org.thp.thehive.dto.{Description, String16, String512}
import play.api.libs.json.{Json, OFormat, OWrites}

import java.util.Date
import be.venneborg.refined.play.RefinedJsonFormats._

case class InputDashboard(title: String, description: String, status: String, definition: String)
case class InputDashboard(
title: String512,
description: Description,
status: String16,
definition: Description
)

object InputDashboard {
implicit val writes: OWrites[InputDashboard] = Json.writes[InputDashboard]
Expand Down
7 changes: 6 additions & 1 deletion dto/src/main/scala/org/thp/thehive/dto/v0/Log.scala
Original file line number Diff line number Diff line change
@@ -1,11 +1,16 @@
package org.thp.thehive.dto.v0

import org.thp.scalligraph.controllers.FFile
import org.thp.thehive.dto.Description
import play.api.libs.json.{Json, OFormat}

import java.util.Date

case class InputLog(message: String, startDate: Option[Date] = None, attachment: Option[FFile] = None)
case class InputLog(
message: Description,
startDate: Option[Date] = None,
attachment: Option[FFile] = None
)

case class OutputLog(
_id: String,
Expand Down
Loading

0 comments on commit 140539c

Please sign in to comment.