Skip to content

Commit 666849a

Browse files
ecktereckter
committed
core: aggregate errors that happen too often during infra loading
Signed-off-by: Eloi Charpentier <[email protected]>
1 parent 675d279 commit 666849a

File tree

5 files changed

+147
-89
lines changed

5 files changed

+147
-89
lines changed

‎core/kt-osrd-rjs-parser/src/main/kotlin/fr/sncf/osrd/RawInfraRJSParser.kt

+9-3
Original file line numberDiff line numberDiff line change
@@ -403,7 +403,11 @@ private fun parseRjsTrackSection(
403403
}
404404
}
405405

406-
fun parseRjsElectrification(builder: RawInfraBuilder, electrification: RJSElectrification) {
406+
fun parseRjsElectrification(
407+
builder: RawInfraBuilder,
408+
electrification: RJSElectrification,
409+
electrificationConflictAggregator: LogAggregator
410+
) {
407411
for (electrificationRange in electrification.trackRanges) {
408412
val applyElectrificationForChunkBetween =
409413
{ chunk: TrackChunkDescriptor, chunkLower: Distance, chunkUpper: Distance ->
@@ -414,7 +418,7 @@ fun parseRjsElectrification(builder: RawInfraBuilder, electrification: RJSElectr
414418
previousElectrification.value != electrification.voltage &&
415419
previousElectrification.value != ""
416420
) {
417-
logger.warn(
421+
electrificationConflictAggregator.registerError(
418422
"Electrification conflict on track-range ${electrificationRange.trackSectionID}" +
419423
"[${previousElectrification.lower + chunk.offset.distance}, " +
420424
"${previousElectrification.upper + chunk.offset.distance}]: " +
@@ -756,9 +760,11 @@ fun parseRJSInfra(rjsInfra: RJSInfra): RawInfra {
756760
}
757761

758762
// Parse electrifications
763+
val electrificationConflictAggregator = LogAggregator({ logger.warn(it) })
759764
for (electrification in rjsInfra.electrifications) {
760-
parseRjsElectrification(builder, electrification)
765+
parseRjsElectrification(builder, electrification, electrificationConflictAggregator)
761766
}
767+
electrificationConflictAggregator.logAggregatedSummary()
762768

763769
for (neutralSection in rjsInfra.neutralSections) {
764770
parseNeutralRanges(builder, false, neutralSection)

‎core/kt-osrd-signaling/src/main/kotlin/fr/sncf/osrd/signaling/impl/BlockBuilder.kt

+77-74
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package fr.sncf.osrd.signaling.impl
33
import fr.sncf.osrd.sim_infra.api.*
44
import fr.sncf.osrd.sim_infra.impl.BlockInfraBuilder
55
import fr.sncf.osrd.sim_infra.impl.blockInfraBuilder
6+
import fr.sncf.osrd.utils.LogAggregator
67
import fr.sncf.osrd.utils.indexing.IdxMap
78
import fr.sncf.osrd.utils.indexing.MutableStaticIdxArrayList
89
import fr.sncf.osrd.utils.units.*
@@ -18,86 +19,87 @@ internal fun internalBuildBlocks(
1819
// Step 1) associate DirDetectorIds to a list of delimiting logical signals
1920
val signalDelimiters = findSignalDelimiters(rawSignalingInfra, loadedSignalInfra)
2021
val detectorEntrySignals = makeDetectorEntrySignals(loadedSignalInfra, signalDelimiters)
21-
return blockInfraBuilder(loadedSignalInfra, rawSignalingInfra) {
22-
// Step 2) iterate on zone paths along the route path.
23-
// - maintain a list of currently active blocks
24-
// - At each signal, add it to compatible current blocks.
25-
// - if the signal is delimiting, stop and create the block (deduplicate it too)
26-
for (route in rawSignalingInfra.routes) {
27-
val routeEntryDet = rawSignalingInfra.getRouteEntry(route)
28-
val routeExitDet = rawSignalingInfra.getRouteExit(route)
29-
val entrySignals = detectorEntrySignals[routeEntryDet]
30-
var currentBlocks =
31-
getInitPartialBlocks(
32-
sigModuleManager,
33-
rawSignalingInfra,
34-
loadedSignalInfra,
35-
entrySignals,
36-
routeEntryDet
37-
)
38-
// while inside the route, we maintain a list of currently active blocks.
39-
// each block either expect any signaling system (when starting from a buffer stop or
40-
// wildcard
41-
// signal),
42-
// or expects a given signaling system. blocks can therefore tell whether a signal
43-
// belongs
44-
// there.
45-
// if a signal is not part of a block, it is ignored
46-
// if a signal delimits a block, it ends the block and starts a new ones, one per driver
47-
// if a signal does not delimit a block and has a single driver, it continues the block
48-
// if a signal does not delimit a block and has multiple drivers, it duplicates the
49-
// block
22+
val missingSignalLogAggregator = LogAggregator({ logger.debug(it) })
23+
val result =
24+
blockInfraBuilder(loadedSignalInfra, rawSignalingInfra) {
25+
// Step 2) iterate on zone paths along the route path.
26+
// - maintain a list of currently active blocks
27+
// - At each signal, add it to compatible current blocks.
28+
// - if the signal is delimiting, stop and create the block (deduplicate it too)
29+
for (route in rawSignalingInfra.routes) {
30+
val routeEntryDet = rawSignalingInfra.getRouteEntry(route)
31+
val routeExitDet = rawSignalingInfra.getRouteExit(route)
32+
val entrySignals = detectorEntrySignals[routeEntryDet]
33+
var currentBlocks =
34+
getInitPartialBlocks(
35+
sigModuleManager,
36+
rawSignalingInfra,
37+
loadedSignalInfra,
38+
entrySignals,
39+
routeEntryDet,
40+
missingSignalLogAggregator,
41+
)
42+
// While inside the route, we maintain a list of currently active blocks. Each block
43+
// either expect any signaling system (when starting from a buffer stop or wildcard
44+
// signal), or expects a given signaling system. Blocks can therefore tell whether a
45+
// signal belongs there.
46+
// If a signal is not part of a block, it is ignored. If a signal delimits a block,
47+
// it ends the block and starts a new ones, one per driver. If a signal does not
48+
// delimit a block and has a single driver, it continues the block. If a signal does
49+
// not delimit a block and has multiple drivers, it duplicates the block.
5050

51-
for (zonePath in rawSignalingInfra.getRoutePath(route)) {
52-
val zonePathLength = rawSignalingInfra.getZonePathLength(zonePath)
53-
for (block in currentBlocks) block.addZonePath(zonePath, zonePathLength)
51+
for (zonePath in rawSignalingInfra.getRoutePath(route)) {
52+
val zonePathLength = rawSignalingInfra.getZonePathLength(zonePath)
53+
for (block in currentBlocks) block.addZonePath(zonePath, zonePathLength)
5454

55-
// iterate over signals which are between the block entry and the block exit
56-
val signals = rawSignalingInfra.getSignals(zonePath)
57-
val signalsPositions = rawSignalingInfra.getSignalPositions(zonePath)
58-
for ((physicalSignal, position) in signals.zip(signalsPositions)) {
59-
val distanceToZonePathEnd = zonePathLength - position
60-
assert(distanceToZonePathEnd >= Distance.ZERO)
61-
assert(distanceToZonePathEnd <= zonePathLength.distance)
62-
for (signal in loadedSignalInfra.getLogicalSignals(physicalSignal)) {
63-
currentBlocks =
64-
updatePartialBlocks(
65-
sigModuleManager,
66-
currentBlocks,
67-
loadedSignalInfra,
68-
signal,
69-
distanceToZonePathEnd,
70-
)
55+
// iterate over signals which are between the block entry and the block exit
56+
val signals = rawSignalingInfra.getSignals(zonePath)
57+
val signalsPositions = rawSignalingInfra.getSignalPositions(zonePath)
58+
for ((physicalSignal, position) in signals.zip(signalsPositions)) {
59+
val distanceToZonePathEnd = zonePathLength - position
60+
assert(distanceToZonePathEnd >= Distance.ZERO)
61+
assert(distanceToZonePathEnd <= zonePathLength.distance)
62+
for (signal in loadedSignalInfra.getLogicalSignals(physicalSignal)) {
63+
currentBlocks =
64+
updatePartialBlocks(
65+
sigModuleManager,
66+
currentBlocks,
67+
loadedSignalInfra,
68+
signal,
69+
distanceToZonePathEnd,
70+
)
71+
}
7172
}
7273
}
73-
}
7474

75-
// when a route ends at a buffer stop, unterminated blocks are expected,
76-
// as the buffer stop sort of acts as a closed signal. when a route does not
77-
// end with a buffer stop, blocks are expected to end with the route.
78-
// such blocks are not valid, and can be fixed by adding a delimiter signal
79-
// right before the end of the route.
80-
val routeEndsAtBufferStop = rawSignalingInfra.isBufferStop(routeExitDet.value)
81-
for (curBlock in currentBlocks) {
82-
if (curBlock.zonePaths.size == 0) continue
83-
if (curBlock.signals.size == 0) continue
75+
// when a route ends at a buffer stop, unterminated blocks are expected,
76+
// as the buffer stop sort of acts as a closed signal. when a route does not
77+
// end with a buffer stop, blocks are expected to end with the route.
78+
// such blocks are not valid, and can be fixed by adding a delimiter signal
79+
// right before the end of the route.
80+
val routeEndsAtBufferStop = rawSignalingInfra.isBufferStop(routeExitDet.value)
81+
for (curBlock in currentBlocks) {
82+
if (curBlock.zonePaths.size == 0) continue
83+
if (curBlock.signals.size == 0) continue
8484

85-
val lastZonePath = curBlock.zonePaths[curBlock.zonePaths.size - 1]
86-
assert(routeExitDet == rawSignalingInfra.getZonePathExit(lastZonePath))
87-
if (!routeEndsAtBufferStop)
88-
logger.debug {
89-
"unterminated block at end of route ${rawSignalingInfra.getRouteName(route)}"
90-
}
91-
block(
92-
curBlock.startAtBufferStop,
93-
true,
94-
curBlock.zonePaths,
95-
curBlock.signals,
96-
curBlock.signalPositions
97-
)
85+
val lastZonePath = curBlock.zonePaths[curBlock.zonePaths.size - 1]
86+
assert(routeExitDet == rawSignalingInfra.getZonePathExit(lastZonePath))
87+
if (!routeEndsAtBufferStop)
88+
logger.debug {
89+
"unterminated block at end of route ${rawSignalingInfra.getRouteName(route)}"
90+
}
91+
block(
92+
curBlock.startAtBufferStop,
93+
true,
94+
curBlock.zonePaths,
95+
curBlock.signals,
96+
curBlock.signalPositions
97+
)
98+
}
9899
}
99100
}
100-
}
101+
missingSignalLogAggregator.logAggregatedSummary()
102+
return result
101103
}
102104

103105
data class AssociatedDetector(val detector: DirDetectorId, val distance: Distance)
@@ -210,14 +212,15 @@ private fun getInitPartialBlocks(
210212
loadedSignalInfra: LoadedSignalInfra,
211213
entrySignals: IdxMap<SignalingSystemId, AssociatedSignal>?,
212214
entryDet: DirDetectorId,
215+
missingSignalLogAggregator: LogAggregator,
213216
): MutableList<PartialBlock> {
214217
val initialBlocks = mutableListOf<PartialBlock>()
215218
val isBufferStop = rawSignalingInfra.isBufferStop(entryDet.value)
216219
if (entrySignals == null) {
217220
if (!isBufferStop)
218-
logger.debug {
221+
missingSignalLogAggregator.registerError(
219222
"no signal at non buffer stop ${rawSignalingInfra.getDetectorName(entryDet.value)}:${entryDet.direction}"
220-
}
223+
)
221224
initialBlocks.add(
222225
PartialBlock(
223226
true,

‎core/kt-osrd-signaling/src/main/kotlin/fr/sncf/osrd/signaling/impl/SignalingSimulatorImpl.kt

+18-9
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import fr.sncf.osrd.signaling.*
44
import fr.sncf.osrd.sim_infra.api.*
55
import fr.sncf.osrd.sim_infra.impl.SignalParameters
66
import fr.sncf.osrd.sim_infra.impl.loadedSignalInfra
7+
import fr.sncf.osrd.utils.LogAggregator
78
import fr.sncf.osrd.utils.indexing.*
89
import fr.sncf.osrd.utils.units.Distance
910
import mu.KotlinLogging
@@ -83,6 +84,14 @@ class SignalingSimulatorImpl(override val sigModuleManager: SigSystemManager) :
8384
loadedSignalInfra: LoadedSignalInfra
8485
): BlockInfra {
8586
val blockInfra = internalBuildBlocks(sigModuleManager, rawSignalingInfra, loadedSignalInfra)
87+
val blockLogAggregator =
88+
LogAggregator(
89+
{ logger.debug(it) },
90+
)
91+
val signalLogAggregator =
92+
LogAggregator(
93+
{ logger.debug(it) },
94+
)
8695
for (block in blockInfra.blocks) {
8796
val sigSystem = blockInfra.getBlockSignalingSystem(block)
8897
val path = blockInfra.getBlockPath(block)
@@ -110,19 +119,17 @@ class SignalingSimulatorImpl(override val sigModuleManager: SigSystemManager) :
110119
val reporter =
111120
object : BlockDiagReporter {
112121
override fun reportBlock(errorType: String) {
113-
logger.debug {
114-
val entrySignal = rawSignalingInfra.getLogicalSignalName(signals[0])
115-
val exitSignal =
116-
rawSignalingInfra.getLogicalSignalName(signals[signals.size - 1])
122+
val entrySignal = rawSignalingInfra.getLogicalSignalName(signals[0])
123+
val exitSignal =
124+
rawSignalingInfra.getLogicalSignalName(signals[signals.size - 1])
125+
blockLogAggregator.registerError(
117126
"error in block from $entrySignal to $exitSignal: $errorType"
118-
}
127+
)
119128
}
120129

121130
override fun reportSignal(sigIndex: Int, errorType: String) {
122-
logger.debug {
123-
val signal = rawSignalingInfra.getLogicalSignalName(signals[sigIndex])
124-
"error at signal $signal: $errorType"
125-
}
131+
val signal = rawSignalingInfra.getLogicalSignalName(signals[sigIndex])
132+
signalLogAggregator.registerError("error at signal $signal: $errorType")
126133
}
127134
}
128135
sigModuleManager.checkSignalingSystemBlock(reporter, sigSystem, sigBlock)
@@ -151,6 +158,8 @@ class SignalingSimulatorImpl(override val sigModuleManager: SigSystemManager) :
151158
)
152159
}
153160
}
161+
blockLogAggregator.logAggregatedSummary()
162+
signalLogAggregator.logAggregatedSummary()
154163
return blockInfra
155164
}
156165

‎core/kt-osrd-utils/src/main/kotlin/fr/sncf/osrd/utils/LogAggregator.kt

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
package fr.sncf.osrd.utils
2+
3+
/**
4+
* This class can be used to aggregate logs: when a specific error/warning happens thousands of time
5+
* in a row, we only report the first $n and only log the total error number afterward. The class
6+
* must be initialized once before the loop, errors should be logged with `logError`, and then
7+
* `logAggregatedSummary` should be called once at the end.
8+
*/
9+
data class LogAggregator(
10+
/** Function to use to log anything (e.g. `{ logger.warn(it) }` ). */
11+
val logFunction: (str: String) -> Unit,
12+
/** String to be used for collapsed errors, using %d and .format for the remaining number. */
13+
val summaryErrorMessage: String = "... and %d other similar errors",
14+
/** Max number of errors before collapsing the rest. */
15+
val maxReportedErrors: Int = 3,
16+
) {
17+
private var nErrors = 0
18+
private var savedErrors = mutableListOf<String>()
19+
20+
/** Registers an error. Does not log anything before the `reportSummary` call. */
21+
fun registerError(msg: String) {
22+
nErrors++
23+
if (savedErrors.size < maxReportedErrors) savedErrors.add(msg)
24+
}
25+
26+
/** Logs the errors, collapsing the ones after `maxReportedErrors`. */
27+
fun logAggregatedSummary() {
28+
for (err in savedErrors) logFunction(err)
29+
val remainingErrors = nErrors - savedErrors.size
30+
if (remainingErrors > 0) logFunction(summaryErrorMessage.format(remainingErrors))
31+
}
32+
}

‎core/src/main/kotlin/fr/sncf/osrd/api/api_v2/RequirementsParser.kt

+11-3
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import fr.sncf.osrd.api.api_v2.conflicts.WorkSchedulesRequest
55
import fr.sncf.osrd.conflicts.*
66
import fr.sncf.osrd.sim_infra.api.RawSignalingInfra
77
import fr.sncf.osrd.standalone_sim.result.ResultTrain
8+
import fr.sncf.osrd.utils.LogAggregator
89
import fr.sncf.osrd.utils.units.Duration
910
import fr.sncf.osrd.utils.units.TimeDelta
1011
import fr.sncf.osrd.utils.units.seconds
@@ -99,9 +100,12 @@ fun convertWorkScheduleMap(
99100
timeToAdd: TimeDelta = 0.seconds
100101
): Collection<Requirements> {
101102
val res = mutableListOf<Requirements>()
103+
val logAggregator = LogAggregator({ requirementsParserLogger.warn(it) })
102104
for (entry in workSchedules) {
103105
val workScheduleRequirements = mutableListOf<ResultTrain.SpacingRequirement>()
104-
workScheduleRequirements.addAll(convertWorkSchedule(rawInfra, entry.value, timeToAdd))
106+
workScheduleRequirements.addAll(
107+
convertWorkSchedule(rawInfra, entry.value, timeToAdd, logAggregator)
108+
)
105109
res.add(
106110
Requirements(
107111
RequirementId(entry.key, RequirementType.WORK_SCHEDULE),
@@ -122,9 +126,12 @@ fun convertWorkScheduleCollection(
122126
workSchedules: Collection<WorkSchedule>,
123127
timeToAdd: TimeDelta = 0.seconds,
124128
): Requirements {
129+
val logAggregator = LogAggregator({ requirementsParserLogger.warn(it) })
125130
val workSchedulesRequirements = mutableListOf<ResultTrain.SpacingRequirement>()
126131
for (workSchedule in workSchedules) {
127-
workSchedulesRequirements.addAll(convertWorkSchedule(rawInfra, workSchedule, timeToAdd))
132+
workSchedulesRequirements.addAll(
133+
convertWorkSchedule(rawInfra, workSchedule, timeToAdd, logAggregator)
134+
)
128135
}
129136
return Requirements(
130137
RequirementId(DEFAULT_WORK_SCHEDULE_ID, RequirementType.WORK_SCHEDULE),
@@ -137,6 +144,7 @@ private fun convertWorkSchedule(
137144
rawInfra: RawSignalingInfra,
138145
workSchedule: WorkSchedule,
139146
timeToAdd: TimeDelta = 0.seconds,
147+
logAggregator: LogAggregator,
140148
): Collection<ResultTrain.SpacingRequirement> {
141149
val res = mutableListOf<ResultTrain.SpacingRequirement>()
142150

@@ -181,7 +189,7 @@ private fun convertWorkSchedule(
181189
"${tracksNotCoveredByRoutes.size} track sections were not fully covered by routes (ignoring some work schedules): " +
182190
tracksNotCoveredByRoutes.take(3).joinToString(", ") +
183191
(if (tracksNotCoveredByRoutes.size > 3) ", ..." else "")
184-
requirementsParserLogger.warn(msg)
192+
logAggregator.registerError(msg)
185193
}
186194
return res
187195
}

0 commit comments

Comments
 (0)