#1731 Wrap graph to include database instance

TheHive-Project · Jan 5, 2021 · 4e3e544 · 4e3e544
1 parent 38dd963
commit 4e3e544
Show file tree

Hide file tree

Showing 86 changed files with 257 additions and 572 deletions.
diff --git a/ScalliGraph b/ScalliGraph
diff --git a/conf/logback.xml b/conf/logback.xml
@@ -36,6 +36,8 @@
 
     <logger name="org.thp.scalligraph.models" level="TRACE"/>
     <logger name="org.thp.scalligraph.traversal" level="TRACE"/>
+    <logger name="org.thp.thehive.services.StreamSrv" level="INFO"/>
+    <logger name="org.thp.thehive.services.StreamActor" level="INFO"/>
     <!--
     <logger name="org.janusgraph.graphdb" level="INFO" />
     <logger name="org.thp.thehive.client" level="DEBUG" />

diff --git a/cortex/connector/src/main/resources/play/reference-overrides.conf b/cortex/connector/src/main/resources/play/reference-overrides.conf
@@ -1,12 +1,12 @@
 akka {
   actor {
     serializers {
-      cortex-schema-updater = "org.thp.thehive.connector.cortex.models.SchemaUpdaterSerializer"
+      //cortex-schema-updater = "org.thp.thehive.connector.cortex.models.SchemaUpdaterSerializer"
       cortex-jobs = "org.thp.thehive.connector.cortex.services.CortexSerializer"
     }
 
     serialization-bindings {
-      "org.thp.thehive.connector.cortex.models.SchemaUpdaterMessage" = cortex-schema-updater
+      //"org.thp.thehive.connector.cortex.models.SchemaUpdaterMessage" = cortex-schema-updater
       "org.thp.thehive.connector.cortex.services.CortexActorMessage" = cortex-jobs
     }
   }

diff --git a/.../src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala b/.../src/main/scala/org/thp/thehive/connector/cortex/controllers/v0/CortexQueryExecutor.scala
@@ -44,7 +44,7 @@ class CortexQueryExecutor @Inject() (
     case tpe if SubType(tpe, ru.typeOf[Traversal.V[Job]]) => ru.typeOf[Traversal.V[Observable]]
   }
 
-  override val customFilterQuery: FilterQuery = FilterQuery(db, publicProperties) { (tpe, globalParser) =>
+  override val customFilterQuery: FilterQuery = FilterQuery(publicProperties) { (tpe, globalParser) =>
     FieldsParser("parentChildFilter") {
       case (_, FObjOne("_parent", ParentIdFilter(_, parentId))) if parentTypes.isDefinedAt(tpe) =>
         Good(new CortexParentIdInputFilter(parentId))
@@ -60,7 +60,6 @@ class CortexQueryExecutor @Inject() (
 
 class CortexParentIdInputFilter(parentId: String) extends InputQuery[Traversal.Unk, Traversal.Unk] {
   override def apply(
-      db: Database,
       publicProperties: PublicProperties,
       traversalType: ru.Type,
       traversal: Traversal.Unk,
@@ -78,15 +77,13 @@ class CortexParentIdInputFilter(parentId: String) extends InputQuery[Traversal.U
   */
 class CortexParentQueryInputFilter(parentFilter: InputQuery[Traversal.Unk, Traversal.Unk]) extends InputQuery[Traversal.Unk, Traversal.Unk] {
   override def apply(
-      db: Database,
       publicProperties: PublicProperties,
       traversalType: ru.Type,
       traversal: Traversal.Unk,
       authContext: AuthContext
   ): Traversal.Unk = {
     def filter[F, T: ru.TypeTag](t: Traversal.V[F] => Traversal.V[T]): Traversal.Unk =
       parentFilter(
-        db,
         publicProperties,
         ru.typeOf[Traversal.V[T]],
         t(traversal.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk],
@@ -100,15 +97,13 @@ class CortexParentQueryInputFilter(parentFilter: InputQuery[Traversal.Unk, Trave
 class CortexChildQueryInputFilter(childType: String, childFilter: InputQuery[Traversal.Unk, Traversal.Unk])
     extends InputQuery[Traversal.Unk, Traversal.Unk] {
   override def apply(
-      db: Database,
       publicProperties: PublicProperties,
       traversalType: ru.Type,
       traversal: Traversal.Unk,
       authContext: AuthContext
   ): Traversal.Unk = {
     def filter[F, T: ru.TypeTag](t: Traversal.V[F] => Traversal.V[T]): Traversal.Unk =
       childFilter(
-        db,
         publicProperties,
         ru.typeOf[Traversal.V[T]],
         t(traversal.asInstanceOf[Traversal.V[F]]).asInstanceOf[Traversal.Unk],

diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Action.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/models/Action.scala
@@ -1,10 +1,9 @@
 package org.thp.thehive.connector.cortex.models
 
 import java.util.Date
-
-import org.apache.tinkerpop.gremlin.structure.{Edge, Graph, Vertex}
+import org.apache.tinkerpop.gremlin.structure.{Edge, Vertex}
 import org.thp.scalligraph.models._
-import org.thp.scalligraph.traversal.Converter
+import org.thp.scalligraph.traversal.{Converter, Graph}
 import org.thp.scalligraph.{BuildVertexEntity, EntityId}
 import play.api.libs.json.JsObject
 
@@ -67,7 +66,7 @@ object ActionContext extends HasModel {
         override def _createdAt: Date           = entity._createdAt
         override def _updatedAt: Option[Date]   = entity._updatedAt
       }
-    override def create(e: ActionContext, from: Vertex, to: Vertex)(implicit db: Database, graph: Graph): Edge =
+    override def create(e: ActionContext, from: Vertex, to: Vertex)(implicit graph: Graph): Edge =
       from.addEdge(label, to)
   }
 }
diff --git a/...connector/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterActor.scala b/...connector/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterActor.scala
diff --git a/...ctor/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterSerializer.scala b/...ctor/src/main/scala/org/thp/thehive/connector/cortex/models/SchemaUpdaterSerializer.scala
diff --git a/...nnector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala b/...nnector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionOperationSrv.scala
@@ -1,11 +1,10 @@
 package org.thp.thehive.connector.cortex.services
 
 import java.util.Date
-
 import javax.inject.Inject
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models.Entity
+import org.thp.scalligraph.traversal.Graph
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{EntityIdOrName, InternalError}
 import org.thp.thehive.connector.cortex.models._

diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/ActionSrv.scala
@@ -1,18 +1,18 @@
 package org.thp.thehive.connector.cortex.services
 
 import java.util.{Date, Map => JMap}
-
 import akka.actor.ActorRef
 import com.google.inject.name.Named
+
 import javax.inject.Inject
-import org.apache.tinkerpop.gremlin.structure.{Element, Graph}
+import org.apache.tinkerpop.gremlin.structure.Element
 import org.thp.cortex.client.CortexClient
 import org.thp.cortex.dto.v0.{InputAction => CortexAction, OutputJob => CortexJob}
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models._
 import org.thp.scalligraph.services._
 import org.thp.scalligraph.traversal.TraversalOps._
-import org.thp.scalligraph.traversal.{Converter, Traversal}
+import org.thp.scalligraph.traversal.{Converter, Graph, Traversal}
 import org.thp.scalligraph.{EntityId, NotFoundError}
 import org.thp.thehive.connector.cortex.controllers.v0.Conversion._
 import org.thp.thehive.connector.cortex.models._

diff --git a/...nector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerTemplateSrv.scala b/...nector/src/main/scala/org/thp/thehive/connector/cortex/services/AnalyzerTemplateSrv.scala
@@ -1,15 +1,14 @@
 package org.thp.thehive.connector.cortex.services
 
 import java.util.zip.{ZipEntry, ZipFile}
-
 import com.google.inject.name.Named
+
 import javax.inject.{Inject, Singleton}
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models.{Database, Entity}
 import org.thp.scalligraph.query.PropertyUpdater
 import org.thp.scalligraph.services._
-import org.thp.scalligraph.traversal.Traversal
+import org.thp.scalligraph.traversal.{Graph, Traversal}
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{CreateError, EntityIdOrName, EntityName}
 import org.thp.thehive.connector.cortex.controllers.v0.Conversion._

diff --git a/...x/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexAuditSrv.scala b/...x/connector/src/main/scala/org/thp/thehive/connector/cortex/services/CortexAuditSrv.scala
@@ -14,9 +14,9 @@ import org.thp.thehive.services.{AuditSrv, UserSrv}
 class CortexAuditSrv @Inject() (
     userSrvProvider: Provider[UserSrv],
     @Named("notification-actor") notificationActor: ActorRef,
-    eventSrv: EventSrv
-)(implicit @Named("with-thehive-cortex-schema") db: Database)
-    extends AuditSrv(userSrvProvider, notificationActor, eventSrv) {
+    eventSrv: EventSrv,
+    db: Database
+) extends AuditSrv(userSrvProvider, notificationActor, eventSrv, db) {
 
   val job              = new ObjectAudit[Job, Observable]
   val action           = new ObjectAudit[Action, Product with Entity]

diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/EntityHelper.scala
@@ -1,9 +1,9 @@
 package org.thp.thehive.connector.cortex.services
 
 import javax.inject.{Inject, Singleton}
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.auth.{AuthContext, Permission}
 import org.thp.scalligraph.models.Entity
+import org.thp.scalligraph.traversal.Graph
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{BadRequestError, EntityIdOrName}
 import org.thp.thehive.models._

diff --git a/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala b/cortex/connector/src/main/scala/org/thp/thehive/connector/cortex/services/JobSrv.scala
@@ -2,24 +2,23 @@ package org.thp.thehive.connector.cortex.services
 
 import java.nio.file.Files
 import java.util.{Date, Map => JMap}
-
 import akka.Done
 import akka.actor._
 import akka.stream.Materializer
 import akka.stream.scaladsl.FileIO
 import com.google.inject.name.Named
 import io.scalaland.chimney.dsl._
+
 import javax.inject.{Inject, Singleton}
 import org.apache.tinkerpop.gremlin.process.traversal.P
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.cortex.client.CortexClient
-import org.thp.cortex.dto.v0.{InputArtifact, OutputArtifact, Attachment => CortexAttachment, OutputJob => CortexJob, JobStatus => CortexJobStatus}
+import org.thp.cortex.dto.v0.{InputArtifact, OutputArtifact, Attachment => CortexAttachment, JobStatus => CortexJobStatus, OutputJob => CortexJob}
 import org.thp.scalligraph.auth.{AuthContext, Permission}
 import org.thp.scalligraph.controllers.FFile
 import org.thp.scalligraph.models.{Database, Entity}
 import org.thp.scalligraph.services._
 import org.thp.scalligraph.traversal.TraversalOps._
-import org.thp.scalligraph.traversal.{Converter, StepLabel, Traversal}
+import org.thp.scalligraph.traversal.{Converter, Graph, StepLabel, Traversal}
 import org.thp.scalligraph.{EntityId, EntityIdOrName, NotFoundError}
 import org.thp.thehive.connector.cortex.controllers.v0.Conversion._
 import org.thp.thehive.connector.cortex.models._

diff --git a/.../scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunAnalyzer.scala b/.../scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunAnalyzer.scala
@@ -1,8 +1,8 @@
 package org.thp.thehive.connector.cortex.services.notification.notifiers
 
 import javax.inject.{Inject, Singleton}
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.models.Entity
+import org.thp.scalligraph.traversal.Graph
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{BadConfigurationError, NotFoundError, RichOption}
 import org.thp.thehive.connector.cortex.services.{AnalyzerSrv, JobSrv}

diff --git a/...scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunResponder.scala b/...scala/org/thp/thehive/connector/cortex/services/notification/notifiers/RunResponder.scala
@@ -1,9 +1,10 @@
 package org.thp.thehive.connector.cortex.services.notification.notifiers
 
 import com.typesafe.config.ConfigRenderOptions
+
 import javax.inject.{Inject, Singleton}
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.models.Entity
+import org.thp.scalligraph.traversal.Graph
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{BadConfigurationError, NotFoundError, RichOption}
 import org.thp.thehive.connector.cortex.services.{ActionSrv, ResponderSrv}

diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/NoAuditSrv.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/NoAuditSrv.scala
@@ -2,11 +2,12 @@ package org.thp.thehive.migration.th4
 
 import akka.actor.ActorRef
 import com.google.inject.name.Named
+
 import javax.inject.{Inject, Provider, Singleton}
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models.{Database, Entity}
 import org.thp.scalligraph.services.EventSrv
+import org.thp.scalligraph.traversal.Graph
 import org.thp.thehive.models.Audit
 import org.thp.thehive.services.{AuditSrv, UserSrv}
 

diff --git a/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala b/migration/src/main/scala/org/thp/thehive/migration/th4/Output.scala
@@ -6,13 +6,12 @@ import com.google.inject.Guice
 import javax.inject.{Inject, Named, Provider, Singleton}
 import net.codingwell.scalaguice.ScalaModule
 import org.apache.tinkerpop.gremlin.process.traversal.P
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.scalligraph._
 import org.thp.scalligraph.auth.{AuthContext, AuthContextImpl, UserSrv => UserDB}
 import org.thp.scalligraph.janus.JanusDatabase
 import org.thp.scalligraph.models.{Database, Entity, Schema, UMapping}
 import org.thp.scalligraph.services._
-import org.thp.scalligraph.traversal.Traversal
+import org.thp.scalligraph.traversal.{Graph, Traversal}
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.thehive.connector.cortex.models.{CortexSchemaDefinition, TheHiveCortexSchemaProvider}
 import org.thp.thehive.connector.cortex.services.{ActionSrv, JobSrv}

diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispExportSrv.scala
@@ -1,12 +1,11 @@
 package org.thp.thehive.connector.misp.services
 
 import java.util.Date
-
 import javax.inject.{Inject, Named, Singleton}
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.misp.dto.{Attribute, Tag => MispTag}
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.models.{Database, Entity}
+import org.thp.scalligraph.traversal.Graph
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.{AuthorizationError, BadRequestError, NotFoundError}
 import org.thp.thehive.models._

diff --git a/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala b/misp/connector/src/main/scala/org/thp/thehive/connector/misp/services/MispImportSrv.scala
@@ -4,11 +4,11 @@ import akka.stream.Materializer
 import akka.stream.scaladsl.{FileIO, Sink, Source}
 import akka.util.ByteString
 import org.apache.tinkerpop.gremlin.process.traversal.P
-import org.apache.tinkerpop.gremlin.structure.Graph
 import org.thp.misp.dto.{Attribute, Event, Tag => MispTag}
 import org.thp.scalligraph.auth.AuthContext
 import org.thp.scalligraph.controllers.FFile
 import org.thp.scalligraph.models._
+import org.thp.scalligraph.traversal.Graph
 import org.thp.scalligraph.traversal.TraversalOps._
 import org.thp.scalligraph.utils.FunctionalCondition._
 import org.thp.scalligraph.{EntityName, RichSeq}
+1 −1		build.sbt
+2 −2		core-test/src/test/scala/org/thp/scalligraph/controllers/FieldsParserMacroTest.scala
+1 −1		core-test/src/test/scala/org/thp/scalligraph/models/CallbackTest.scala
+3 −3		core-test/src/test/scala/org/thp/scalligraph/models/CardinalityTest.scala
+1 −1		core-test/src/test/scala/org/thp/scalligraph/models/DatabaseProviders.scala
+4 −5		core-test/src/test/scala/org/thp/scalligraph/models/Modern.scala
+3 −3		core-test/src/test/scala/org/thp/scalligraph/models/SimpleEntityTest.scala
+1 −1		core-test/src/test/scala/org/thp/scalligraph/services/IntegrityCheckTest.scala
+5 −0		core/src/main/scala/org/thp/scalligraph/SingleInstance.scala
+1 −1		core/src/main/scala/org/thp/scalligraph/controllers/Entrypoint.scala
+6 −5		core/src/main/scala/org/thp/scalligraph/macro/ModelMacro.scala
+36 −0		core/src/main/scala/org/thp/scalligraph/macro/TraversalMacro.scala
+35 −25		core/src/main/scala/org/thp/scalligraph/models/Database.scala
+48 −10		core/src/main/scala/org/thp/scalligraph/models/Mapping.scala
+7 −8		core/src/main/scala/org/thp/scalligraph/models/Model.scala
+9 −16		core/src/main/scala/org/thp/scalligraph/models/Operation.scala
+18 −15		core/src/main/scala/org/thp/scalligraph/models/Schema.scala
+12 −21		core/src/main/scala/org/thp/scalligraph/query/Aggregation.scala
+12 −22		core/src/main/scala/org/thp/scalligraph/query/Filter.scala
+27 −18		core/src/main/scala/org/thp/scalligraph/query/InputSort.scala
+27 −0		core/src/main/scala/org/thp/scalligraph/query/PredicateOps.scala
+115 −138		core/src/main/scala/org/thp/scalligraph/query/PropertyBuilder.scala
+101 −46		core/src/main/scala/org/thp/scalligraph/query/PublicProperty.scala
+9 −16		core/src/main/scala/org/thp/scalligraph/query/Query.scala
+5 −6		core/src/main/scala/org/thp/scalligraph/query/QueryExecutor.scala
+11 −17		core/src/main/scala/org/thp/scalligraph/services/EdgeSrv.scala
+9 −3		core/src/main/scala/org/thp/scalligraph/services/ElementSrv.scala
+2 −2		core/src/main/scala/org/thp/scalligraph/services/IntegrityCheckOps.scala
+2 −3		core/src/main/scala/org/thp/scalligraph/services/StorageSrv.scala
+16 −20		core/src/main/scala/org/thp/scalligraph/services/VertexSrv.scala
+15 −15		core/src/main/scala/org/thp/scalligraph/services/config/ConfigItem.scala
+18 −20		core/src/main/scala/org/thp/scalligraph/services/config/ContextConfigItem.scala
+1 −1		core/src/main/scala/org/thp/scalligraph/traversal/Converter.scala
+42 −0		core/src/main/scala/org/thp/scalligraph/traversal/Graph.scala
+26 −0		core/src/main/scala/org/thp/scalligraph/traversal/GraphStrategy.scala
+42 −0		core/src/main/scala/org/thp/scalligraph/traversal/OptimizeIndexStrategy.scala
+46 −0		core/src/main/scala/org/thp/scalligraph/traversal/RewriteOrderGlobalStepStrategy.scala
+3 −1		core/src/main/scala/org/thp/scalligraph/traversal/Selectors.scala
+32 −15		core/src/main/scala/org/thp/scalligraph/traversal/Traversal.scala
+21 −12		core/src/main/scala/org/thp/scalligraph/traversal/TraversalOps.scala
+167 −0		core/src/main/scala/org/thp/scalligraph/traversal/TraversalPrinter.scala
+7 −4		core/src/main/scala/org/thp/scalligraph/utils/Retry.scala
+43 −0		database/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/ElementValueComparatorAcceptNull.java
+62 −0		database/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/IndexOptimizerStrategy.java
+81 −0		database/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/JanusGraphAcceptNullStrategy.java
+267 −0		database/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/JanusGraphStepAcceptNull.java
+31 −0		database/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/LimitedIterator.java
+85 −0		...e/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/MultiDistinctOrderedIteratorAcceptNull.java
+112 −0		database/janusgraph/src/main/java/org/thp/scalligraph/janus/strategies/RewriteOrderGlobalStepStrategy.java
+5 −2		database/janusgraph/src/main/resources/reference.conf
+60 −0		database/janusgraph/src/main/scala/org/thp/scalligraph/janus/JanusClusterManagerActor.scala
+248 −152		database/janusgraph/src/main/scala/org/thp/scalligraph/janus/JanusDatabase.scala
+142 −0		database/janusgraph/src/main/scala/org/thp/scalligraph/janus/JanusDatabaseProvider.scala
+1 −1		database/neo4j/src/main/scala/org/thp/scalligraph/neo4j/Neo4jDatabase.scala
+1 −1		database/orientdb/src/main/scala/org/thp/scalligraph/orientdb/OrientDatabase.scala
+1 −0		project/Dependencies.scala