Skip to content

Commit 2476e1d

Browse files
wangtaohzshidayang
authored andcommitted
[AMORO-2253] Mixed Format optimized-sequence should not contains the skipped partitions (#2249)
(cherry picked from commit 4f70e5e)
1 parent 25237fb commit 2476e1d

File tree

1 file changed

+10
-9
lines changed

1 file changed

+10
-9
lines changed

ams/server/src/main/java/com/netease/arctic/server/optimizing/plan/OptimizingPlanner.java

+10-9
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,8 @@ public class OptimizingPlanner extends OptimizingEvaluator {
5151
private final PartitionPlannerFactory partitionPlannerFactory;
5252
private List<TaskDescriptor> tasks;
5353

54+
private List<AbstractPartitionPlan> actualPartitionPlans;
55+
5456
public OptimizingPlanner(TableRuntime tableRuntime, ArcticTable table, double availableCore) {
5557
super(tableRuntime, table);
5658
this.partitionFilter =
@@ -69,17 +71,17 @@ protected PartitionEvaluator buildEvaluator(String partitionPath) {
6971
}
7072

7173
public Map<String, Long> getFromSequence() {
72-
return partitionPlanMap.entrySet().stream()
74+
return actualPartitionPlans.stream()
7375
.collect(
7476
Collectors.toMap(
75-
Map.Entry::getKey, e -> ((AbstractPartitionPlan) e.getValue()).getFromSequence()));
77+
AbstractPartitionPlan::getPartition, AbstractPartitionPlan::getFromSequence));
7678
}
7779

7880
public Map<String, Long> getToSequence() {
79-
return partitionPlanMap.entrySet().stream()
81+
return actualPartitionPlans.stream()
8082
.collect(
8183
Collectors.toMap(
82-
Map.Entry::getKey, e -> ((AbstractPartitionPlan) e.getValue()).getToSequence()));
84+
AbstractPartitionPlan::getPartition, AbstractPartitionPlan::getToSequence));
8385
}
8486

8587
@Override
@@ -126,10 +128,10 @@ public List<TaskDescriptor> planTasks() {
126128
evaluators.sort(Comparator.comparing(PartitionEvaluator::getWeight, Comparator.reverseOrder()));
127129

128130
double maxInputSize = MAX_INPUT_FILE_SIZE_PER_THREAD * availableCore;
129-
List<PartitionEvaluator> inputPartitions = Lists.newArrayList();
131+
actualPartitionPlans = Lists.newArrayList();
130132
long actualInputSize = 0;
131133
for (PartitionEvaluator evaluator : evaluators) {
132-
inputPartitions.add(evaluator);
134+
actualPartitionPlans.add((AbstractPartitionPlan) evaluator);
133135
actualInputSize += evaluator.getCost();
134136
if (actualInputSize > maxInputSize) {
135137
break;
@@ -138,9 +140,8 @@ public List<TaskDescriptor> planTasks() {
138140

139141
double avgThreadCost = actualInputSize / availableCore;
140142
List<TaskDescriptor> tasks = Lists.newArrayList();
141-
for (PartitionEvaluator evaluator : inputPartitions) {
142-
tasks.addAll(
143-
((AbstractPartitionPlan) evaluator).splitTasks((int) (actualInputSize / avgThreadCost)));
143+
for (AbstractPartitionPlan partitionPlan : actualPartitionPlans) {
144+
tasks.addAll(partitionPlan.splitTasks((int) (actualInputSize / avgThreadCost)));
144145
}
145146
if (!tasks.isEmpty()) {
146147
if (evaluators.stream()

0 commit comments

Comments
 (0)