diff --git a/CHANGELOG.md b/CHANGELOG.md index 48989a670e..335d9618fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Change Log +## [4.1.1](https://github.com/TheHive-Project/TheHive/milestone/70) (2021-03-23) + +**Implemented enhancements:** + +- [Feature Request] Include organisation ID in webhooks [\#1865](https://github.com/TheHive-Project/TheHive/issues/1865) + +**Closed issues:** + +- [Bug] Importing the ATT&CK library fails on 4.1 [\#1862](https://github.com/TheHive-Project/TheHive/issues/1862) +- Thehive4.1.0 Issues with Lucene [\#1863](https://github.com/TheHive-Project/TheHive/issues/1863) + +**Fixed bugs:** + +- [Bug] TheHive doesn't start if webhook is configured without authentication [\#1859](https://github.com/TheHive-Project/TheHive/issues/1859) +- [Bug] Migration fails from 4.0.5 to 4.1 [\#1861](https://github.com/TheHive-Project/TheHive/issues/1861) +- [Bug] Filter by "IMPORTED" does not work [\#1866](https://github.com/TheHive-Project/TheHive/issues/1866) +- [Bug] TheHive doesn't start in cluster mode (serializer is missing) [\#1868](https://github.com/TheHive-Project/TheHive/issues/1868) +- [Bug] Full-text search is slow [\#1870](https://github.com/TheHive-Project/TheHive/issues/1870) + ## [4.1.0](https://github.com/TheHive-Project/TheHive/milestone/56) (2021-03-18) **Implemented enhancements:** diff --git a/ScalliGraph b/ScalliGraph index b403c2c2db..e9122723b8 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit b403c2c2db7e163550022283512c1f6d0c9fe91a +Subproject commit e9122723b83fd87a4c1e808fe5fbe3d5626ef2ad diff --git a/build.sbt b/build.sbt index 2c0db1876a..94da0fb5b8 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.1.0-1" +val thehiveVersion = "4.1.1-1" val scala212 = "2.12.13" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/frontend/bower.json b/frontend/bower.json index 458235d50e..51d23b2778 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.0-1", + "version": "4.1.1-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index d41920c98a..f681cd92f0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.0-1", + "version": "4.1.1-1", "license": "AGPL-3.0", "repository": { "type": "git", diff --git a/package/docker/entrypoint b/package/docker/entrypoint index 2f355f0668..bd64d6fa0a 100755 --- a/package/docker/entrypoint +++ b/package/docker/entrypoint @@ -1,6 +1,7 @@ #!/bin/bash CQL_HOSTNAMES=${TH_CQL_HOSTNAMES:-cassandra} BDB_DIRECTORY=${TH_BDB_DIRECTORY:-/data/db} +INDEX_DIRECTORY=${TH_INDEX_DIRECTORY:-/data/index} HDFS_URL=${TH_HDFS_URL} STORAGE_DIRECTORY=${TH_STORAGE_DIRECTORY:-/data/files} test "${TH_NO_CONFIG_SECRET}" == 1 @@ -60,6 +61,7 @@ do "--cql-username") shift; CQL_USERNAME=$1 ;; "--cql-password") shift; CQL_PASSWORD=$1 ;; "--bdb-directory") shift; BDB_DIRECTORY=$1 ;; + "--index-directory") shift; INDEX_DIRECTORY=$1 ;; "--no-config-storage") CONFIG_STORAGE=0 ;; "--hdfs-url") shift; HDFS_URL=$1 ;; "--storage-directory") shift; STORAGE_DIRECTORY=$1 ;; @@ -103,6 +105,12 @@ then echo "storage.backend = berkeleyje" >> ${CONFIG_FILE} echo "storage.directory = \"${BDB_DIRECTORY}\"" >> ${CONFIG_FILE} echo "berkeleyje.freeDisk = 1" >> ${CONFIG_FILE} + if test -e "${BDB_DIRECTORY}" + then + test -w "${BDB_DIRECTORY}" || echo "WARNING the directory used to store database ($BDB_DIRECTORY) is not writable" + else + mkdir -p "${BDB_DIRECTORY}" || echo "WARNING the directory used to store database ($BDB_DIRECTORY) is not writable" + fi else echo "Using cassandra address = ${CQL[@]}" echo "storage.backend = cql" >> ${CONFIG_FILE} @@ -120,6 +128,14 @@ then echo "Waiting until Cassandra DB is up" sleep 30 # Sleep until cassandra Db is up fi + echo "index.search.backend = lucene" >> ${CONFIG_FILE} + echo "index.search.directory = \"${INDEX_DIRECTORY}\"" >> ${CONFIG_FILE} + if test -e "${INDEX_DIRECTORY}" + then + test -w "${INDEX_DIRECTORY}" || echo "WARNING the directory used to store index ($INDEX_DIRECTORY) is not writable" + else + mkdir -p "${INDEX_DIRECTORY}" || echo "WARNING the directory used to store index ($INDEX_DIRECTORY) is not writable" + fi echo "}" >> ${CONFIG_FILE} fi @@ -140,6 +156,12 @@ then mkdir -p "${STORAGE_DIRECTORY}" echo "provider: localfs" >> ${CONFIG_FILE} echo "localfs.directory: \"${STORAGE_DIRECTORY}\"" >> ${CONFIG_FILE} + if test -e "${STORAGE_DIRECTORY}" + then + test -w "${STORAGE_DIRECTORY}" || echo "WARNING the directory used to store files ($STORAGE_DIRECTORY) is not writable" + else + mkdir -p "${STORAGE_DIRECTORY}" || echo "WARNING the directory used to store files ($STORAGE_DIRECTORY) is not writable" + fi fi echo "}" >> ${CONFIG_FILE} fi diff --git a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala index e8da33fce3..931676302d 100644 --- a/thehive/app/org/thp/thehive/controllers/v1/Properties.scala +++ b/thehive/app/org/thp/thehive/controllers/v1/Properties.scala @@ -51,7 +51,7 @@ class Properties @Inject() ( .property("_id", UMapping.entityId)( _.select(_._id) .filter[EntityId] { - case (_, t, _, Right(p)) => t.has(T.id, p.map(_.value)) + case (_, t, _, Right(p)) => t.has(T.id, p.mapValue(_.value)) case (_, t, _, Left(true)) => t case (_, t, _, _) => t.empty } diff --git a/thehive/app/org/thp/thehive/models/Alert.scala b/thehive/app/org/thp/thehive/models/Alert.scala index 9a4dc54f03..4520dfd231 100644 --- a/thehive/app/org/thp/thehive/models/Alert.scala +++ b/thehive/app/org/thp/thehive/models/Alert.scala @@ -43,7 +43,7 @@ case class AlertTag() @DefineIndex(IndexType.standard, "source") @DefineIndex(IndexType.standard, "sourceRef") @DefineIndex(IndexType.fulltext, "title") -@DefineIndex(IndexType.fulltext, "description") +@DefineIndex(IndexType.fulltextOnly, "description") @DefineIndex(IndexType.standard, "severity") @DefineIndex(IndexType.standard, "date") @DefineIndex(IndexType.standard, "lastSyncDate") diff --git a/thehive/app/org/thp/thehive/models/Case.scala b/thehive/app/org/thp/thehive/models/Case.scala index bb3408bd1f..2d12f92754 100644 --- a/thehive/app/org/thp/thehive/models/Case.scala +++ b/thehive/app/org/thp/thehive/models/Case.scala @@ -83,7 +83,7 @@ case class CaseProcedure() @BuildVertexEntity @DefineIndex(IndexType.unique, "number") @DefineIndex(IndexType.fulltext, "title") -@DefineIndex(IndexType.fulltext, "description") +@DefineIndex(IndexType.fulltextOnly, "description") @DefineIndex(IndexType.standard, "severity") @DefineIndex(IndexType.standard, "startDate") @DefineIndex(IndexType.standard, "endDate") @@ -91,7 +91,7 @@ case class CaseProcedure() @DefineIndex(IndexType.standard, "tlp") @DefineIndex(IndexType.standard, "pap") @DefineIndex(IndexType.standard, "status") -@DefineIndex(IndexType.fulltext, "summary") +@DefineIndex(IndexType.fulltextOnly, "summary") @DefineIndex(IndexType.standard, "tags") @DefineIndex(IndexType.standard, "assignee") @DefineIndex(IndexType.standard, "organisationIds") diff --git a/thehive/app/org/thp/thehive/models/Log.scala b/thehive/app/org/thp/thehive/models/Log.scala index f1f48bf5f2..aad8f6b651 100644 --- a/thehive/app/org/thp/thehive/models/Log.scala +++ b/thehive/app/org/thp/thehive/models/Log.scala @@ -8,7 +8,7 @@ import java.util.Date @BuildEdgeEntity[Log, Attachment] case class LogAttachment() -@DefineIndex(IndexType.fulltext, "message") +@DefineIndex(IndexType.fulltextOnly, "message") @DefineIndex(IndexType.standard, "date") @DefineIndex(IndexType.standard, "taskId") @DefineIndex(IndexType.standard, "organisationIds") diff --git a/thehive/app/org/thp/thehive/models/Observable.scala b/thehive/app/org/thp/thehive/models/Observable.scala index 8b3201d41f..aa39712f10 100644 --- a/thehive/app/org/thp/thehive/models/Observable.scala +++ b/thehive/app/org/thp/thehive/models/Observable.scala @@ -17,7 +17,7 @@ case class ObservableData() @BuildEdgeEntity[Observable, Tag] case class ObservableTag() -@DefineIndex(IndexType.fulltext, "message") +@DefineIndex(IndexType.fulltextOnly, "message") @DefineIndex(IndexType.standard, "tlp") @DefineIndex(IndexType.standard, "ioc") @DefineIndex(IndexType.standard, "sighted") diff --git a/thehive/app/org/thp/thehive/models/Tag.scala b/thehive/app/org/thp/thehive/models/Tag.scala index 2acb23cc0f..ee98cc4f0e 100644 --- a/thehive/app/org/thp/thehive/models/Tag.scala +++ b/thehive/app/org/thp/thehive/models/Tag.scala @@ -4,7 +4,8 @@ import org.thp.scalligraph.BuildVertexEntity import org.thp.scalligraph.models.{DefineIndex, IndexType} @DefineIndex(IndexType.unique, "namespace", "predicate", "value") -@DefineIndex(IndexType.fulltext, "namespace", "predicate", "value", "description") +@DefineIndex(IndexType.fulltext, "namespace", "predicate", "value") +@DefineIndex(IndexType.fulltextOnly, "description") @BuildVertexEntity case class Tag( namespace: String, diff --git a/thehive/app/org/thp/thehive/models/Task.scala b/thehive/app/org/thp/thehive/models/Task.scala index 300f2f20e4..335580a0a2 100644 --- a/thehive/app/org/thp/thehive/models/Task.scala +++ b/thehive/app/org/thp/thehive/models/Task.scala @@ -21,7 +21,7 @@ case class TaskLog() @BuildVertexEntity @DefineIndex(IndexType.fulltext, "title") @DefineIndex(IndexType.standard, "group") -@DefineIndex(IndexType.fulltext, "description") +@DefineIndex(IndexType.fulltextOnly, "description") @DefineIndex(IndexType.standard, "status") @DefineIndex(IndexType.standard, "flag") @DefineIndex(IndexType.standard, "startDate") diff --git a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala index e5e0e55460..3f1c57023a 100644 --- a/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala +++ b/thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala @@ -391,6 +391,27 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema { traversal.outE("OrganisationDashboard").raw.property("writable", true).iterate() Success(()) } + //=====[release 4.1.0]===== + .removeIndex("Alert", IndexType.fulltext, "description") + .removeIndex("Case", IndexType.fulltext, "description", "summary") + .removeIndex("Log", IndexType.fulltext, "message") + .removeIndex("Observable", IndexType.fulltext, "message") + .removeIndex("Log", IndexType.fulltext, "message") + .removeIndex("Tag", IndexType.fulltext, "description") + .removeIndex("Task", IndexType.fulltext, "description") + .updateGraph("Set caseId in imported alerts", "Alert") { traversal => + traversal + .project( + _.by + .by(_.out("AlertCase")._id.option) + ) + .foreach { + case (vertex, caseId) => + caseId.foreach(vertex.property("caseId", _)) + case _ => + } + Success(()) + } val reflectionClasses = new Reflections( new ConfigurationBuilder() diff --git a/thehive/app/org/thp/thehive/services/AlertSrv.scala b/thehive/app/org/thp/thehive/services/AlertSrv.scala index c72a62c876..e237b30997 100644 --- a/thehive/app/org/thp/thehive/services/AlertSrv.scala +++ b/thehive/app/org/thp/thehive/services/AlertSrv.scala @@ -262,6 +262,7 @@ class AlertSrv @Inject() ( createdCase <- caseSrv.create(case0, assignee, organisation, customField, caseTemplate, Nil) _ <- importObservables(alert.alert, createdCase.`case`) _ <- alertCaseSrv.create(AlertCase(), alert.alert, createdCase.`case`) + _ <- get(alert.alert).update(_.caseId, Some(createdCase._id)).getOrFail("Alert") _ <- markAsRead(alert._id) _ = integrityCheckActor ! EntityAdded("Alert") } yield createdCase @@ -481,11 +482,11 @@ object AlertOps { .value(_.`type`) .headOption .map { - case CustomFieldType.boolean => traversal.filter(_.customFields(customField).has(_.booleanValue, predicate.map(_.as[Boolean]))) - case CustomFieldType.date => traversal.filter(_.customFields(customField).has(_.dateValue, predicate.map(_.as[Date]))) - case CustomFieldType.float => traversal.filter(_.customFields(customField).has(_.floatValue, predicate.map(_.as[Double]))) - case CustomFieldType.integer => traversal.filter(_.customFields(customField).has(_.integerValue, predicate.map(_.as[Int]))) - case CustomFieldType.string => traversal.filter(_.customFields(customField).has(_.stringValue, predicate.map(_.as[String]))) + case CustomFieldType.boolean => traversal.filter(_.customFields(customField).has(_.booleanValue, predicate.mapValue(_.as[Boolean]))) + case CustomFieldType.date => traversal.filter(_.customFields(customField).has(_.dateValue, predicate.mapValue(_.as[Date]))) + case CustomFieldType.float => traversal.filter(_.customFields(customField).has(_.floatValue, predicate.mapValue(_.as[Double]))) + case CustomFieldType.integer => traversal.filter(_.customFields(customField).has(_.integerValue, predicate.mapValue(_.as[Int]))) + case CustomFieldType.string => traversal.filter(_.customFields(customField).has(_.stringValue, predicate.mapValue(_.as[String]))) } .getOrElse(traversal.empty) diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index 0f86e0b7ed..094978ef53 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -507,11 +507,11 @@ object CaseOps { .value(_.`type`) .headOption .map { - case CustomFieldType.boolean => traversal.filter(_.customFields(customField).has(_.booleanValue, predicate.map(_.as[Boolean]))) - case CustomFieldType.date => traversal.filter(_.customFields(customField).has(_.dateValue, predicate.map(_.as[Date]))) - case CustomFieldType.float => traversal.filter(_.customFields(customField).has(_.floatValue, predicate.map(_.as[Double]))) - case CustomFieldType.integer => traversal.filter(_.customFields(customField).has(_.integerValue, predicate.map(_.as[Int]))) - case CustomFieldType.string => traversal.filter(_.customFields(customField).has(_.stringValue, predicate.map(_.as[String]))) + case CustomFieldType.boolean => traversal.filter(_.customFields(customField).has(_.booleanValue, predicate.mapValue(_.as[Boolean]))) + case CustomFieldType.date => traversal.filter(_.customFields(customField).has(_.dateValue, predicate.mapValue(_.as[Date]))) + case CustomFieldType.float => traversal.filter(_.customFields(customField).has(_.floatValue, predicate.mapValue(_.as[Double]))) + case CustomFieldType.integer => traversal.filter(_.customFields(customField).has(_.integerValue, predicate.mapValue(_.as[Int]))) + case CustomFieldType.string => traversal.filter(_.customFields(customField).has(_.stringValue, predicate.mapValue(_.as[String]))) } .getOrElse(traversal.empty) diff --git a/thehive/app/org/thp/thehive/services/IntegrityCheckSerializer.scala b/thehive/app/org/thp/thehive/services/IntegrityCheckSerializer.scala index 1e9cfb7114..23b62e3369 100644 --- a/thehive/app/org/thp/thehive/services/IntegrityCheckSerializer.scala +++ b/thehive/app/org/thp/thehive/services/IntegrityCheckSerializer.scala @@ -1,6 +1,7 @@ package org.thp.thehive.services import akka.serialization.Serializer +import play.api.libs.json.{Json, OFormat} import java.io.NotSerializableException @@ -9,12 +10,19 @@ class IntegrityCheckSerializer extends Serializer { override def includeManifest: Boolean = false + implicit val duplicationCheckResultFormat: OFormat[DuplicationCheckResult] = Json.format[DuplicationCheckResult] + implicit val globalCheckResultFormat: OFormat[GlobalCheckResult] = Json.format[GlobalCheckResult] + override def toBinary(o: AnyRef): Array[Byte] = o match { - case EntityAdded(name) => 0.toByte +: name.getBytes - case NeedCheck(name) => 1.toByte +: name.getBytes - case DuplicationCheck(name) => 2.toByte +: name.getBytes - case _ => throw new NotSerializableException + case EntityAdded(name) => 0.toByte +: name.getBytes + case NeedCheck(name) => 1.toByte +: name.getBytes + case DuplicationCheck(name) => 2.toByte +: name.getBytes + case duplicationCheckResult: DuplicationCheckResult => 3.toByte +: Json.toJson(duplicationCheckResult).toString.getBytes + case GlobalCheckRequest(name) => 4.toByte +: name.getBytes + case globalCheckResult: GlobalCheckResult => 5.toByte +: Json.toJson(globalCheckResult).toString.getBytes + case GetCheckStats(name) => 6.toByte +: name.getBytes + case _ => throw new NotSerializableException } override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef = @@ -22,6 +30,10 @@ class IntegrityCheckSerializer extends Serializer { case 0 => EntityAdded(new String(bytes.tail)) case 1 => NeedCheck(new String(bytes.tail)) case 2 => DuplicationCheck(new String(bytes.tail)) + case 3 => Json.parse(bytes.tail).as[DuplicationCheckResult] + case 4 => GlobalCheckRequest(new String(bytes.tail)) + case 5 => Json.parse(bytes.tail).as[GlobalCheckResult] + case 6 => GetCheckStats(new String(bytes.tail)) case _ => throw new NotSerializableException } } diff --git a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala index 69db24a794..718d33a36d 100644 --- a/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala +++ b/thehive/app/org/thp/thehive/services/notification/notifiers/Webhook.scala @@ -2,7 +2,7 @@ package org.thp.thehive.services.notification.notifiers import akka.stream.Materializer import org.apache.tinkerpop.gremlin.structure.Vertex -import org.thp.client.{Authentication, ProxyWS, ProxyWSConfig} +import org.thp.client.{Authentication, NoAuthentication, ProxyWS, ProxyWSConfig} import org.thp.scalligraph.models.{Entity, UMapping} import org.thp.scalligraph.services.config.{ApplicationConfig, ConfigItem} import org.thp.scalligraph.traversal.TraversalOps._ @@ -31,7 +31,7 @@ case class WebhookNotification( name: String, url: String, version: Int = 0, - auth: Authentication, + auth: Authentication = NoAuthentication, wsConfig: ProxyWSConfig = ProxyWSConfig(), includedTheHiveOrganisations: Seq[String] = Seq("*"), excludedTheHiveOrganisations: Seq[String] = Nil @@ -258,7 +258,11 @@ class Webhook( else { val ws = new ProxyWS(config.wsConfig, mat) val async = for { - message <- Future.fromTry(buildMessage(config.version, audit)) + message <- Future.fromTry( + buildMessage(config.version, audit).map( + _ + ("organisationId" -> JsString(organisation._id.toString)) + ("organisation" -> JsString(organisation.name)) + ) + ) _ = logger.debug(s"Request webhook with message $message") resp <- config.auth(ws.url(config.url)).post(message) } yield if (resp.status >= 400) logger.warn(s"Webhook call on ${config.url} returns ${resp.status} ${resp.statusText}") else () diff --git a/thehive/conf/play/reference-overrides.conf b/thehive/conf/play/reference-overrides.conf index a4d40ea45d..1c0cb430ae 100644 --- a/thehive/conf/play/reference-overrides.conf +++ b/thehive/conf/play/reference-overrides.conf @@ -9,6 +9,10 @@ play.filters { ] } +play.http.parser.maxDiskBuffer = 128MB +play.http.parser.maxMemoryBuffer = 256kB + + # Register module for dependency injection play.modules.enabled += org.thp.thehive.TheHiveModule diff --git a/thehive/test/org/thp/thehive/controllers/v0/StatusCtrlTest.scala b/thehive/test/org/thp/thehive/controllers/v0/StatusCtrlTest.scala index 77121cd741..1557f0ddeb 100644 --- a/thehive/test/org/thp/thehive/controllers/v0/StatusCtrlTest.scala +++ b/thehive/test/org/thp/thehive/controllers/v0/StatusCtrlTest.scala @@ -70,7 +70,7 @@ class StatusCtrlTest extends PlaySpecification with TestAppBuilder { "pollingDuration" -> 1000 ), "schemaStatus" -> Json.arr( - Json.obj("name" -> "thehive", "currentVersion" -> 59, "expectedVersion" -> 59, "error" -> JsNull) + Json.obj("name" -> "thehive", "currentVersion" -> 67, "expectedVersion" -> 67, "error" -> JsNull) ) )