Skip to content

Commit 59becf3

Browse files
nscuromehab
authored andcommitted
Improve BOM processing performance and make it transactional (#218)
* Add bloated BOM for ingestion performance testing Signed-off-by: nscuro <[email protected]> * Prevent query compilation cache being bypassed for `matchSingleIdentity` queries See DependencyTrack/dependency-track#2540 This also cleans the query from containing weird statements like `(cpe != null && cpe == null)` in case a component does not have a CPE. Signed-off-by: nscuro <[email protected]> * WIP: Improve BOM processing performance Signed-off-by: nscuro <[email protected]> * Handle dependency graph Signed-off-by: nscuro <[email protected]> * Improve dependency graph assembly Instead of using individual bulk UPDATE queries, use setters on persistent components instead. This way we can again make use of batched flushing. Signed-off-by: nscuro <[email protected]> * Completely replace old processing logic Also decompose large processing method into multiple smaller ones, and re-implement notifications. Signed-off-by: nscuro <[email protected]> * Fix not all BOM refs being updated with new component identities Signed-off-by: nscuro <[email protected]> * Be smarter about indexing component identities and BOM refs Also add more documentation Signed-off-by: nscuro <[email protected]> * Reduce logging noise Signed-off-by: nscuro <[email protected]> * Mark new components as such ... via new transient field. Required for compatibility with #217 Signed-off-by: nscuro <[email protected]> * Compatibility with #217 Signed-off-by: nscuro <[email protected]> * Cleanup tests Signed-off-by: nscuro <[email protected]> * Reduce code duplication Signed-off-by: nscuro <[email protected]> * Cleanup; Process services Signed-off-by: nscuro <[email protected]> * Finishing touches 🪄 Signed-off-by: nscuro <[email protected]> * Make flush threshold configurable The optimal value could depend on how beefy the database server is, and how much memory is available to the API server. Signed-off-by: nscuro <[email protected]> * Clarify `warn` log when rolling back active transactions Signed-off-by: nscuro <[email protected]> * Log number of consumed components and services before and after de-dupe Signed-off-by: nscuro <[email protected]> * Extend BOM processing test with bloated BOM Signed-off-by: nscuro <[email protected]> * Make component identity matching strict To address DependencyTrack/dependency-track#2519 (comment). Also add regression test for this specific issue. Signed-off-by: nscuro <[email protected]> * Add regression test for DependencyTrack/dependency-track#1905 Signed-off-by: nscuro <[email protected]> * Clarify why "reachability on commit" is disabled; Add assertion for persistent object state Signed-off-by: nscuro <[email protected]> * Add tests for `equals` and `hashCode` of `ComponentIdentity` Signed-off-by: nscuro <[email protected]> * Address review comments Signed-off-by: nscuro <[email protected]> --------- Signed-off-by: nscuro <[email protected]> Signed-off-by: mehab <[email protected]>
1 parent 2601710 commit 59becf3

22 files changed

+444007
-572
lines changed

src/main/java/org/dependencytrack/common/ConfigKey.java

+2-3
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,6 @@
22

33
import alpine.Config;
44

5-
import java.time.Duration;
6-
75
public enum ConfigKey implements Config.Key {
86

97
SYSTEM_REQUIREMENT_CHECK_ENABLED("system.requirement.check.enabled", true),
@@ -33,7 +31,8 @@ public enum ConfigKey implements Config.Key {
3331
TASK_COMPONENT_IDENTIFICATION_LOCK_AT_MOST_FOR("task.componentIdentification.lockAtMostForInMillis", "900000"),
3432
TASK_COMPONENT_IDENTIFICATION_LOCK_AT_LEAST_FOR("task.componentIdentification.lockAtLeastForInMillis", "3000"),
3533
TASK_LDAP_SYNC_LOCK_AT_MOST_FOR("task.ldapSync.lockAtMostForInMillis", "900000"),
36-
TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR("task.ldapSync.lockAtLeastForInMillis", "3000");
34+
TASK_LDAP_SYNC_LOCK_AT_LEAST_FOR("task.ldapSync.lockAtLeastForInMillis", "3000"),
35+
BOM_UPLOAD_PROCESSING_TRX_FLUSH_THRESHOLD("bom.upload.processing.trx.flush.threshold", "10000");
3736

3837
private final String propertyName;
3938
private final Object defaultValue;

src/main/java/org/dependencytrack/event/BomUploadEvent.java

+6-6
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,9 @@
1919
package org.dependencytrack.event;
2020

2121
import alpine.event.framework.AbstractChainableEvent;
22+
import org.dependencytrack.model.Project;
2223

2324
import java.io.File;
24-
import java.util.UUID;
2525

2626
/**
2727
* Defines an event triggered when a bill-of-material (bom) document is submitted.
@@ -31,16 +31,16 @@
3131
*/
3232
public class BomUploadEvent extends AbstractChainableEvent {
3333

34-
private final UUID projectUuid;
34+
private final Project project;
3535
private final File file;
3636

37-
public BomUploadEvent(final UUID projectUuid, final File file) {
38-
this.projectUuid = projectUuid;
37+
public BomUploadEvent(final Project project, final File file) {
38+
this.project = project;
3939
this.file = file;
4040
}
4141

42-
public UUID getProjectUuid() {
43-
return projectUuid;
42+
public Project getProject() {
43+
return project;
4444
}
4545

4646
public File getFile() {

src/main/java/org/dependencytrack/model/Component.java

+18-2
Original file line numberDiff line numberDiff line change
@@ -352,12 +352,12 @@ public enum FetchGroup {
352352
private UUID uuid;
353353

354354
private transient String bomRef;
355+
private transient String licenseId;
355356
private transient DependencyMetrics metrics;
356357
private transient RepositoryMetaComponent repositoryMeta;
358+
private transient boolean isNew;
357359
private transient int usedBy;
358-
359360
private transient Set<String> dependencyGraph;
360-
361361
private transient boolean expandDependencyGraph;
362362

363363
public long getId() {
@@ -741,6 +741,14 @@ public void setRepositoryMeta(RepositoryMetaComponent repositoryMeta) {
741741
this.repositoryMeta = repositoryMeta;
742742
}
743743

744+
public boolean isNew() {
745+
return isNew;
746+
}
747+
748+
public void setNew(final boolean aNew) {
749+
isNew = aNew;
750+
}
751+
744752
public Double getLastInheritedRiskScore() {
745753
return lastInheritedRiskScore;
746754
}
@@ -757,6 +765,14 @@ public void setBomRef(String bomRef) {
757765
this.bomRef = bomRef;
758766
}
759767

768+
public String getLicenseId() {
769+
return licenseId;
770+
}
771+
772+
public void setLicenseId(final String licenseId) {
773+
this.licenseId = licenseId;
774+
}
775+
760776
public int getUsedBy() {
761777
return usedBy;
762778
}

src/main/java/org/dependencytrack/model/ComponentIdentity.java

+14
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.dependencytrack.util.PurlUtil;
2424
import org.json.JSONObject;
2525

26+
import java.util.Objects;
2627
import java.util.UUID;
2728

2829
/**
@@ -137,6 +138,19 @@ public UUID getUuid() {
137138
return uuid;
138139
}
139140

141+
@Override
142+
public boolean equals(final Object o) {
143+
if (this == o) return true;
144+
if (o == null || getClass() != o.getClass()) return false;
145+
final ComponentIdentity that = (ComponentIdentity) o;
146+
return objectType == that.objectType && Objects.equals(purl, that.purl) && Objects.equals(purlCoordinates, that.purlCoordinates) && Objects.equals(cpe, that.cpe) && Objects.equals(swidTagId, that.swidTagId) && Objects.equals(group, that.group) && Objects.equals(name, that.name) && Objects.equals(version, that.version) && Objects.equals(uuid, that.uuid);
147+
}
148+
149+
@Override
150+
public int hashCode() {
151+
return Objects.hash(objectType, purl, purlCoordinates, cpe, swidTagId, group, name, version, uuid);
152+
}
153+
140154
public JSONObject toJSON() {
141155
final JSONObject jsonObject = new JSONObject();
142156
jsonObject.put("uuid", this.getUuid());

0 commit comments

Comments
 (0)