Skip to content

Commit

Permalink
#2081 Add API for database repair
Browse files Browse the repository at this point in the history
  • Loading branch information
@To-om
To-om committed Jun 13, 2021
1 parent 22b149c commit 8228291
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 11 deletions.
2 changes: 1 addition & 1 deletion ScalliGraph
Submodule ScalliGraph updated 3 files
+1 −1 core/src/main/scala/org/thp/scalligraph/models/Database.scala
+81 −72 core/src/main/scala/org/thp/scalligraph/models/Operation.scala
+20 −10 database/janusgraph/src/main/scala/org/thp/scalligraph/janus/JanusDatabase.scala
90 changes: 87 additions & 3 deletions thehive/app/org/thp/thehive/controllers/v1/AdminCtrl.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ import akka.util.Timeout
import ch.qos.logback.classic.{Level, LoggerContext}
import org.slf4j.LoggerFactory
import org.thp.scalligraph.controllers.Entrypoint
import org.thp.scalligraph.models.Database
import org.thp.scalligraph.models._
import org.thp.scalligraph.services.GenIntegrityCheckOps
import org.thp.thehive.models.Permissions
import org.thp.thehive.services.{CheckState, CheckStats, GetCheckStats, GlobalCheckRequest}
Expand All @@ -18,14 +18,15 @@ import javax.inject.{Inject, Named, Singleton}
import scala.collection.immutable
import scala.concurrent.duration.DurationInt
import scala.concurrent.{ExecutionContext, Future}
import scala.util.Success
import scala.util.{Failure, Success}

@Singleton
class AdminCtrl @Inject() (
entrypoint: Entrypoint,
@Named("integrity-check-actor") integrityCheckActor: ActorRef,
integrityCheckOps: immutable.Set[GenIntegrityCheckOps],
db: Database,
schemas: immutable.Set[UpdatableSchema],
implicit val ec: ExecutionContext
) {

Expand Down Expand Up @@ -113,6 +114,7 @@ class AdminCtrl @Inject() (
"Job",
"Attachment"
)

def indexStatus: Action[AnyContent] =
entrypoint("Get index status")
.authPermittedRoTransaction(db, Permissions.managePlatform) { _ => graph =>
Expand All @@ -122,7 +124,7 @@ class AdminCtrl @Inject() (
catch {
case error: Throwable =>
logger.error("Index fetch error", error)
0
0L
}
Json.obj("name" -> label, "count" -> count)

Expand All @@ -137,4 +139,86 @@ class AdminCtrl @Inject() (
Future(db.reindexData(label))
Success(Results.NoContent)
}

private val rangeRegex = "(\\d+)-(\\d+)".r
private def getOperations(schemaName: String, select: String, filter: String): Seq[(Operation, Int)] = {
val ranges = select match {
case "all" => Seq(0 until Int.MaxValue)
case other =>
other.split(',').toSeq.map {
case rangeRegex(from, to) => from.toInt to to.toInt
case number => number.toInt to number.toInt
}
}

val filters = filter.split(',')

schemas
.filter(_.name == schemaName)
.toSeq
.flatMap { schema =>
schema
.operations
.operations
.zipWithIndex
.filter {
case (_, i) => ranges.exists(_.contains(i + 1))
}
.filter {
case (_: AddVertexModel, _) =>
filters.contains("AddVertexModel") || filters
.contains("schema") || (filters.contains("all") && !filters.contains("!schema") && !filters.contains("!AddVertexModel"))
case (_: AddEdgeModel, _) =>
filter.contains("AddEdgeModel") || filter
.contains("schema") || (filters.contains("all") && !filter.contains("!schema") && !filter.contains("!AddEdgeModel"))
case (_: AddProperty, _) =>
filter.contains("AddProperty") || filter
.contains("schema") || (filters.contains("all") && !filter.contains("!schema") && !filter.contains("!AddProperty"))
case (_: RemoveProperty, _) =>
filter.contains("RemoveProperty") || filter
.contains("schema") || (filters.contains("all") && !filter.contains("!schema") && !filter.contains("!RemoveProperty"))
case (_: UpdateGraph, _) =>
filter.contains("UpdateGraph") || filter
.contains("data") || (filters.contains("all") && !filter.contains("!data") && !filter.contains("!UpdateGraph"))
case (_: AddIndex, _) =>
filter.contains("AddIndex") || filter
.contains("index") || (filters.contains("all") && !filter.contains("!index") && !filter.contains("!AddIndex"))
case (RebuildIndexes, _) =>
filter.contains("RebuildIndexes") || filter
.contains("index") || (filters.contains("all") && !filter.contains("!index") && !filter.contains("!RebuildIndexes"))
case (NoOperation, _) => false
case (_: RemoveIndex, _) =>
filter.contains("RemoveIndex") || filter
.contains("index") || (filters.contains("all") && !filter.contains("!index") && !filter.contains("!RemoveIndex"))
case (_: DBOperation[_], _) =>
filter.contains("DBOperation") || filter
.contains("data") || (filters.contains("all") && !filter.contains("!data") && !filter.contains("!DBOperation"))
}
}
}

def schemaRepair(schemaName: String, select: String, filter: String): Action[AnyContent] =
entrypoint("Repair schema")
.authPermitted(Permissions.managePlatform) { _ =>
val result = getOperations(schemaName, select, filter)
.map {
case (operation, index) =>
logger.info(s"Repair schema: $index=${operation.info}")
operation.execute(db, logger.info(_)) match {
case _: Success[_] => s"${operation.info}: Success"
case Failure(error) => s"${operation.info}: Failure $error"
}
}

Success(Results.Ok(Json.toJson(result)))
}

def schemaInfo(schemaName: String, select: String, filter: String): Action[AnyContent] =
entrypoint("Schema info")
.authPermitted(Permissions.managePlatform) { _ =>
val output = getOperations(schemaName, select, filter).map {
case (o, i) => s"$i=${o.info}"
}
Success(Results.Ok(Json.toJson(output)))
}
}
12 changes: 7 additions & 5 deletions thehive/app/org/thp/thehive/controllers/v1/Router.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,11 +43,13 @@ class Router @Inject() (
case GET(p"/status") => statusCtrl.get
// GET /health controllers.StatusCtrl.health

case GET(p"/admin/check/stats") => adminCtrl.checkStats
case GET(p"/admin/check/$name/trigger") => adminCtrl.triggerCheck(name)
case GET(p"/admin/index/status") => adminCtrl.indexStatus
case GET(p"/admin/index/$name/reindex") => adminCtrl.reindex(name)
case GET(p"/admin/log/set/$packageName/$level") => adminCtrl.setLogLevel(packageName, level)
case GET(p"/admin/check/stats") => adminCtrl.checkStats
case GET(p"/admin/check/$name/trigger") => adminCtrl.triggerCheck(name)
case GET(p"/admin/index/status") => adminCtrl.indexStatus
case GET(p"/admin/index/$name/reindex") => adminCtrl.reindex(name)
case GET(p"/admin/log/set/$packageName/$level") => adminCtrl.setLogLevel(packageName, level)
case POST(p"/admin/schema/repair/$schemaName/$select/$filter") => adminCtrl.schemaRepair(schemaName, select, filter)
case POST(p"/admin/schema/info/$schemaName/$select/$filter") => adminCtrl.schemaInfo(schemaName, select, filter)

// GET /logout controllers.AuthenticationCtrl.logout()
case GET(p"/logout") => authenticationCtrl.logout
Expand Down
4 changes: 2 additions & 2 deletions thehive/app/org/thp/thehive/models/TheHiveSchemaDefinition.scala
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
traversal.removeProperty("colour").iterate()
Success(())
}
.removeProperty("Tag", "colour", usedOnlyByThisModel = true)
.removeProperty[Int]("Tag", "colour", usedOnlyByThisModel = true)
.addProperty[String]("Tag", "colour")
.updateGraph("Add property colour for Tags ", "Tag") { traversal =>
traversal.raw.property("colour", "#000000").iterate()
Expand Down Expand Up @@ -393,7 +393,7 @@ class TheHiveSchemaDefinition @Inject() extends Schema with UpdatableSchema {
traversal.removeProperty("deleted")
Success(())
}
.removeProperty(model = "Log", propertyName = "deleted", usedOnlyByThisModel = true)
.removeProperty[Boolean](model = "Log", propertyName = "deleted", usedOnlyByThisModel = true)
.updateGraph("Make shared dashboard writable", "Dashboard") { traversal =>
traversal.outE("OrganisationDashboard").raw.property("writable", true).iterate()
Success(())
Expand Down

0 comments on commit 8228291

Please sign in to comment.