diff --git a/.gitignore b/.gitignore index 2c87b915d9..90ff884861 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ logs bin conf/application.conf conf/migration.conf +/conf/cloner.conf graphql.config.json graphql.schema.json .graphqlconfig diff --git a/CHANGELOG.md b/CHANGELOG.md index 44b9983ee8..45f298a135 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,17 @@ # Change Log +## [4.1.7](https://github.com/TheHive-Project/TheHive/milestone/76) (2021-07-05) + +**Implemented enhancements:** + +- [Enhancement] Copy the database even if the schema version doesn't match (with force flag) [\#2105](https://github.com/TheHive-Project/TheHive/issues/2105) + +**Fixed bugs:** + +- [Bug] Issue with Migration 3.5.1 -> 4.1.6 [\#2089](https://github.com/TheHive-Project/TheHive/issues/2089) +- [Bug] Fix serialization for case number messages [\#2107](https://github.com/TheHive-Project/TheHive/issues/2107) +- [Bug] Case is removed if the assignee is removed [\#2109](https://github.com/TheHive-Project/TheHive/issues/2109) + ## [4.1.6](https://github.com/TheHive-Project/TheHive/milestone/75) (2021-06-14) **Implemented enhancements:** diff --git a/ScalliGraph b/ScalliGraph index f1a647d4dd..ff5911920a 160000 --- a/ScalliGraph +++ b/ScalliGraph @@ -1 +1 @@ -Subproject commit f1a647d4dd61f538d0d390f30d357e37d6a3aebc +Subproject commit ff5911920aca12e491878970e1a1061fc9998453 diff --git a/build.sbt b/build.sbt index 8db30d29ea..aeeceb03dc 100644 --- a/build.sbt +++ b/build.sbt @@ -2,7 +2,7 @@ import Dependencies._ import com.typesafe.sbt.packager.Keys.bashScriptDefines import org.thp.ghcl.Milestone -val thehiveVersion = "4.1.6-1" +val thehiveVersion = "4.1.7-1" val scala212 = "2.12.13" val scala213 = "2.13.1" val supportedScalaVersions = List(scala212, scala213) diff --git a/conf/application.sample.conf b/conf/application.sample.conf index 4de3da18e0..1045ee97b3 100644 --- a/conf/application.sample.conf +++ b/conf/application.sample.conf @@ -21,6 +21,14 @@ db.janusgraph { keyspace: thehive } } + index.search { + backend: lucene + directory: /opt/thp/thehive/index + # If TheHive is in cluster ElasticSearch must be used: + // backend: elasticsearch + // hostname: ["ip1", "ip2"] + // index-name: thehive + } ## For test only ! # Comment the two lines below before enable Cassandra database diff --git a/conf/cloner.sample.conf b/conf/cloner.sample.conf new file mode 100644 index 0000000000..990147dde4 --- /dev/null +++ b/conf/cloner.sample.conf @@ -0,0 +1,24 @@ +# This is a sample configuration for the database cloner tool + +# Configuration of the source database (same format as in application.conf) +from.db.janusgraph { + storage { + // backend: cql + // hostname: ["ip1", "ip2"] + } + index.search { + backend: lucene + directory: /opt/thp/thehive/index + } +} +# Configuration of the target database +to.db.janusgraph { + storage { + // backend: cql + // hostname: ["ip1", "ip2"] + } + index.search { + backend: lucene + directory: /opt/thp/thehive/otherIndex + } +} diff --git a/frontend/bower.json b/frontend/bower.json index 31b21b6795..8eda9b376d 100644 --- a/frontend/bower.json +++ b/frontend/bower.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.6-1", + "version": "4.1.7-1", "license": "AGPL-3.0", "dependencies": { "jquery": "^3.4.1", diff --git a/frontend/package.json b/frontend/package.json index 1f6fa7f507..d653140327 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,6 +1,6 @@ { "name": "thehive", - "version": "4.1.6-1", + "version": "4.1.7-1", "license": "AGPL-3.0", "repository": { "type": "git", diff --git a/migration/src/main/resources/reference.conf b/migration/src/main/resources/reference.conf index 90a131e6a0..57b1bb0eeb 100644 --- a/migration/src/main/resources/reference.conf +++ b/migration/src/main/resources/reference.conf @@ -117,3 +117,4 @@ to { } } batchSize: 100 +force: false \ No newline at end of file diff --git a/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala b/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala index 5d15b3cadd..6b91022fc9 100644 --- a/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala +++ b/migration/src/main/scala/org/thp/thehive/cloner/Cloner.scala @@ -1,7 +1,7 @@ package org.thp.thehive.cloner import akka.actor.ActorSystem -import com.typesafe.config.{Config, ConfigFactory} +import com.typesafe.config.{Config, ConfigFactory, ConfigValueFactory} import org.apache.tinkerpop.gremlin.structure.T import org.thp.scalligraph.SingleInstance import org.thp.scalligraph.janus.JanusDatabase @@ -31,6 +31,9 @@ object Cloner extends App with IntegrityCheckApp { ) } + def addConfig(config: Config, path: String, value: Any): Config = + config.withValue(path, ConfigValueFactory.fromAnyRef(value)) + val defaultLoggerConfigFile = "/etc/thehive/logback-cloner.xml" if (System.getProperty("logger.file") == null && Files.exists(Paths.get(defaultLoggerConfigFile))) System.setProperty("logger.file", defaultLoggerConfigFile) @@ -54,7 +57,9 @@ object Cloner extends App with IntegrityCheckApp { .valueName("") .required() .action((f, c) => ConfigFactory.parseFileAnySyntax(f).withFallback(c)) - .text("configuration file") + .text("configuration file"), + opt[Unit]('f', "force") + .action((_, c) => addConfig(c, "force", true)) ) } val defaultConfig = @@ -78,19 +83,28 @@ object Cloner extends App with IntegrityCheckApp { val thehiveSchema = new TheHiveSchemaDefinition val cortexSchema = new CortexSchemaDefinition - if (sourceDatabase.version(thehiveSchema.name) != thehiveSchema.operations.operations.length + 1) { - println( - "The schema of TheHive is not valid " + - s"(found ${sourceDatabase.version(thehiveSchema.name)}, expected ${thehiveSchema.operations.operations.length + 1})" - ) - sys.exit(1) + + { + val expectedVersion = thehiveSchema.operations.operations.length + 1 + val foundVersion = sourceDatabase.version(thehiveSchema.name) + if (foundVersion != expectedVersion) { + println(s"The schema of TheHive is not valid (expected: $expectedVersion, found: $foundVersion)") + if (config.getBoolean("force")) + println("Continuing ...") + else + sys.exit(1) + } } - if (sourceDatabase.version(cortexSchema.name) != cortexSchema.operations.operations.length + 1) { - println( - "The schema of Cortex is not valid " + - s"(found ${sourceDatabase.version(cortexSchema.name)}, expected ${cortexSchema.operations.operations.length + 1})" - ) - sys.exit(1) + { + val expectedVersion = cortexSchema.operations.operations.length + 1 + val foundVersion = sourceDatabase.version(cortexSchema.name) + if (foundVersion != expectedVersion) { + println(s"The schema of Cortex is not valid (expected: $expectedVersion, found: $foundVersion)") + if (config.getBoolean("force")) + println("Continuing ...") + else + sys.exit(1) + } } val destDatabase: Database = getDatabase( @@ -111,8 +125,8 @@ object Cloner extends App with IntegrityCheckApp { // don't create initial values val models = destDatabase.extraModels ++ thehiveSchema.modelList ++ cortexSchema.modelList destDatabase.createSchema(models) - destDatabase.setVersion(thehiveSchema.name, thehiveSchema.operations.operations.length + 1) - destDatabase.setVersion(cortexSchema.name, cortexSchema.operations.operations.length + 1) + destDatabase.setVersion(thehiveSchema.name, sourceDatabase.version(thehiveSchema.name)) + destDatabase.setVersion(cortexSchema.name, sourceDatabase.version(cortexSchema.name)) val batchSize: Int = config.getInt("batchSize") @@ -167,6 +181,7 @@ object Cloner extends App with IntegrityCheckApp { println("Add indices ...") destDatabase.addSchemaIndexes(models) + println("Run checks ...") runChecks(destDatabase, Configuration(config)) destDatabase.close() } finally { diff --git a/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala b/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala index 1224be9325..61b4bcfca1 100644 --- a/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala +++ b/migration/src/main/scala/org/thp/thehive/cloner/IntegrityCheckApp.scala @@ -40,6 +40,7 @@ trait IntegrityCheckApp { bindActor[DummyActor]("config-actor") bindActor[DummyActor]("cortex-actor") bindActor[DummyActor]("integrity-check-actor") + bindTypedActor(CaseNumberActor.behavior, "case-number-actor") val integrityCheckOpsBindings = ScalaMultibinder.newSetBinder[GenIntegrityCheckOps](binder) integrityCheckOpsBindings.addBinding.to[ProfileIntegrityCheckOps] diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala index 12a8137888..2585c8e22c 100644 --- a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala +++ b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala @@ -52,6 +52,7 @@ object Output { bindActor[DummyActor]("config-actor") bindActor[DummyActor]("cortex-actor") bindActor[DummyActor]("integrity-check-actor") + bindTypedActor(CaseNumberActor.behavior, "case-number-actor") val schemaBindings = ScalaMultibinder.newSetBinder[UpdatableSchema](binder) schemaBindings.addBinding.to[TheHiveSchemaDefinition] diff --git a/thehive/app/org/thp/thehive/TheHiveModule.scala b/thehive/app/org/thp/thehive/TheHiveModule.scala index 6667d840b2..988e8101b2 100644 --- a/thehive/app/org/thp/thehive/TheHiveModule.scala +++ b/thehive/app/org/thp/thehive/TheHiveModule.scala @@ -107,7 +107,7 @@ class TheHiveModule(environment: Environment, configuration: Configuration) exte integrityCheckOpsBindings.addBinding.to[ObservableIntegrityCheckOps] integrityCheckOpsBindings.addBinding.to[LogIntegrityCheckOps] bind[ActorRef].annotatedWithName("integrity-check-actor").toProvider[IntegrityCheckActorProvider] - bind[TypedActorRef[CaseNumberActor.Request]].annotatedWithName("case-number-actor").toProvider[CaseNumberActorProvider] + bind[TypedActorRef[CaseNumberActor.Request]].toProvider[CaseNumberActorProvider] bind[ActorRef].annotatedWithName("flow-actor").toProvider[FlowActorProvider] diff --git a/thehive/app/org/thp/thehive/services/CaseNumber.scala b/thehive/app/org/thp/thehive/services/CaseNumber.scala index 7366151eae..f25e242bb6 100644 --- a/thehive/app/org/thp/thehive/services/CaseNumber.scala +++ b/thehive/app/org/thp/thehive/services/CaseNumber.scala @@ -1,6 +1,6 @@ package org.thp.thehive.services -import akka.actor.typed.scaladsl.Behaviors +import akka.actor.typed.scaladsl.{ActorContext, Behaviors} import akka.actor.typed.scaladsl.adapter.ClassicActorSystemOps import akka.actor.typed.{ActorRefResolver, Behavior, ActorRef => TypedActorRef} import akka.actor.{ActorSystem, ExtendedActorSystem} @@ -12,6 +12,7 @@ import org.thp.thehive.GuiceAkkaExtension import org.thp.thehive.services.CaseOps._ import java.io.NotSerializableException +import java.nio.ByteBuffer import javax.inject.{Inject, Provider, Singleton} object CaseNumberActor { @@ -21,15 +22,25 @@ object CaseNumberActor { case class GetNextNumber(replyTo: TypedActorRef[Response]) extends Request case class NextNumber(number: Int) extends Response - val behavior: Behavior[Request] = Behaviors.setup[Request] { context => + val behavior: Behavior[Request] = Behaviors.setup[Request](context => waitFirstRequest(context)) + + def getNextCaseNumber(context: ActorContext[Request]): Int = { val injector = GuiceAkkaExtension(context.system).injector val db = injector.getInstance(classOf[Database]) val caseSrv = injector.getInstance(classOf[CaseSrv]) db.roTransaction { implicit graph => - caseNumberProvider(caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1) + caseSrv.startTraversal.getLast.headOption.fold(0)(_.number) + 1 } } + def waitFirstRequest(context: ActorContext[Request]): Behaviors.Receive[Request] = + Behaviors.receiveMessage { + case GetNextNumber(replyTo) => + val nextNumber = getNextCaseNumber(context) + replyTo ! NextNumber(nextNumber) + caseNumberProvider(nextNumber + 1) + } + def caseNumberProvider(nextNumber: Int): Behavior[Request] = Behaviors.receiveMessage { case GetNextNumber(replyTo) => @@ -55,9 +66,8 @@ class CaseNumberSerializer(system: ExtendedActorSystem) extends Serializer { override def toBinary(o: AnyRef): Array[Byte] = o match { case GetNextNumber(replyTo) => 0.toByte +: actorRefResolver.toSerializationFormat(replyTo).getBytes - case NextNumber(number) => - Array(1.toByte, ((number >> 24) % 0xff).toByte, ((number >> 16) % 0xff).toByte, ((number >> 8) % 0xff).toByte, (number % 0xff).toByte) - case _ => throw new NotSerializableException + case NextNumber(number) => ByteBuffer.allocate(5).put(1.toByte).putInt(number).array() + case _ => throw new NotSerializableException } override def includeManifest: Boolean = false @@ -65,12 +75,6 @@ class CaseNumberSerializer(system: ExtendedActorSystem) extends Serializer { override def fromBinary(bytes: Array[Byte], manifest: Option[Class[_]]): AnyRef = bytes(0) match { case 0 => GetNextNumber(actorRefResolver.resolveActorRef(new String(bytes.tail))) - case 1 => - NextNumber( - (bytes(2) << 24) + - (bytes(3) << 16) + - (bytes(4) << 8) + - bytes(5) - ) + case 1 => NextNumber(ByteBuffer.wrap(bytes).getInt(1)) } } diff --git a/thehive/app/org/thp/thehive/services/CaseSrv.scala b/thehive/app/org/thp/thehive/services/CaseSrv.scala index a57cef3a73..683157ff50 100644 --- a/thehive/app/org/thp/thehive/services/CaseSrv.scala +++ b/thehive/app/org/thp/thehive/services/CaseSrv.scala @@ -1,8 +1,9 @@ package org.thp.thehive.services -import akka.actor.ActorRef import akka.actor.typed.scaladsl.AskPattern._ +import akka.actor.typed.scaladsl.adapter.ClassicSchedulerOps import akka.actor.typed.{Scheduler, ActorRef => TypedActorRef} +import akka.actor.{ActorRef, ActorSystem} import akka.util.Timeout import org.apache.tinkerpop.gremlin.process.traversal.{Order, P} import org.apache.tinkerpop.gremlin.structure.Vertex @@ -34,7 +35,7 @@ import java.lang.{Long => JLong} import java.util.{Date, List => JList, Map => JMap} import javax.inject.{Inject, Named, Provider, Singleton} import scala.concurrent.duration.DurationInt -import scala.concurrent.{Await, ExecutionContext, Future} +import scala.concurrent.{Await, ExecutionContextExecutor, Future} import scala.util.{Failure, Success, Try} @Singleton @@ -53,10 +54,9 @@ class CaseSrv @Inject() ( userSrv: UserSrv, alertSrvProvider: Provider[AlertSrv], @Named("integrity-check-actor") integrityCheckActor: ActorRef, - @Named("case-number-actor") caseNumberActor: TypedActorRef[CaseNumberActor.Request], + caseNumberActor: TypedActorRef[CaseNumberActor.Request], cache: SyncCacheApi, - implicit val ec: ExecutionContext, - implicit val scheduler: Scheduler + system: ActorSystem ) extends VertexSrv[Case] { lazy val alertSrv: AlertSrv = alertSrvProvider.get @@ -133,7 +133,9 @@ class CaseSrv @Inject() ( } def nextCaseNumberAsync: Future[Int] = { - implicit val timeout: Timeout = Timeout(1.minute) + implicit val timeout: Timeout = Timeout(1.minute) + implicit val scheduler: Scheduler = system.scheduler.toTyped + implicit val ec: ExecutionContextExecutor = system.dispatcher caseNumberActor.ask[CaseNumberActor.Response](replyTo => CaseNumberActor.GetNextNumber(replyTo)).map { case CaseNumberActor.NextNumber(caseNumber) => caseNumber } diff --git a/thehive/test/org/thp/thehive/services/CaseNumberTest.scala b/thehive/test/org/thp/thehive/services/CaseNumberTest.scala new file mode 100644 index 0000000000..c3caeb7d46 --- /dev/null +++ b/thehive/test/org/thp/thehive/services/CaseNumberTest.scala @@ -0,0 +1,34 @@ +package org.thp.thehive.services + +import akka.actor.ExtendedActorSystem +import akka.actor.typed.ActorSystem +import akka.actor.typed.scaladsl.Behaviors +import org.thp.thehive.TestAppBuilder +import play.api.test.PlaySpecification + +class CaseNumberTest extends PlaySpecification with TestAppBuilder { + + "case number actor" should { + "serialize and deserialize messages" in withActorSystem { system => + val ref = system.deadLetters[CaseNumberActor.Response] + val sut = new CaseNumberSerializer(system.classicSystem.asInstanceOf[ExtendedActorSystem]) + + val messages = Seq( + CaseNumberActor.GetNextNumber(ref), + CaseNumberActor.NextNumber(42), + CaseNumberActor.NextNumber(Int.MaxValue) + ) + val out = messages.map(message => sut.toBinary(message)) + + val result = out.map(bin => sut.fromBinary(bin)) + + result must beEqualTo(messages) + } + } + + private def withActorSystem[T](body: ActorSystem[Nothing] => T) = { + val system = ActorSystem(Behaviors.empty, "test") + try body(system) + finally system.terminate() + } +}