diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt index 261c2061e06..5b3793447bf 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt @@ -82,12 +82,12 @@ Join condition: None Output [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/part] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#9)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#45)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct (12) FilterExecTransformer Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] -Arguments: (((((isnotnull(p_brand#6) AND isnotnull(p_type#7)) AND NOT (p_brand#6 = Brand#9)) AND NOT StartsWith(p_type#7, MEDIUM POLISHED)) AND p_size#8 IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#5)) +Arguments: (((((isnotnull(p_brand#6) AND isnotnull(p_type#7)) AND NOT (p_brand#6 = Brand#45)) AND NOT StartsWith(p_type#7, MEDIUM POLISHED)) AND p_size#8 IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#5)) (13) WholeStageCodegenTransformer (2) Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] @@ -121,19 +121,19 @@ Aggregate Attributes: [] Results [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] (20) ProjectExecTransformer -Output [5]: [hash(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 42) AS hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Output [5]: [hash(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 42) AS hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] (21) WholeStageCodegenTransformer (3) -Input [5]: [hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Arguments: false (22) VeloxResizeBatches -Input [5]: [hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Arguments: 1024, 2147483647, 10485760 (23) ColumnarExchange -Input [5]: [hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 1), ENSURE_REQUIREMENTS, [p_brand#6, p_type#7, p_size#8, ps_suppkey#2], [plan_id=3], [shuffle_writer_type=hash] (24) InputAdapter @@ -153,64 +153,64 @@ Results [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Keys [3]: [p_brand#6, p_type#7, p_size#8] Functions [1]: [partial_count(distinct ps_suppkey#2)] -Aggregate Attributes [1]: [count(ps_suppkey#2)#11] -Results [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Aggregate Attributes [1]: [count(ps_suppkey#2)#10] +Results [4]: [p_brand#6, p_type#7, p_size#8, count#11] (28) ProjectExecTransformer -Output [5]: [hash(p_brand#6, p_type#7, p_size#8, 42) AS hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Output [5]: [hash(p_brand#6, p_type#7, p_size#8, 42) AS hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] (29) WholeStageCodegenTransformer (4) -Input [5]: [hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] +Input [5]: [hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] Arguments: false (30) VeloxResizeBatches -Input [5]: [hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] +Input [5]: [hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] Arguments: 1024, 2147483647, 10485760 (31) ColumnarExchange -Input [5]: [hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] -Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, 1), ENSURE_REQUIREMENTS, [p_brand#6, p_type#7, p_size#8, count#12], [plan_id=4], [shuffle_writer_type=hash] +Input [5]: [hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] +Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, 1), ENSURE_REQUIREMENTS, [p_brand#6, p_type#7, p_size#8, count#11], [plan_id=4], [shuffle_writer_type=hash] (32) InputAdapter -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] (33) InputIteratorTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] (34) RegularHashAggregateExecTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] Keys [3]: [p_brand#6, p_type#7, p_size#8] Functions [1]: [count(distinct ps_suppkey#2)] -Aggregate Attributes [1]: [count(ps_suppkey#2)#11] -Results [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#11 AS supplier_cnt#14] +Aggregate Attributes [1]: [count(ps_suppkey#2)#10] +Results [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#10 AS supplier_cnt#13] (35) WholeStageCodegenTransformer (5) -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] Arguments: false (36) VeloxResizeBatches -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] Arguments: 1024, 2147483647, 10485760 (37) ColumnarExchange -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] -Arguments: rangepartitioning(supplier_cnt#14 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=5], [shuffle_writer_type=hash] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: rangepartitioning(supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=5], [shuffle_writer_type=hash] (38) InputAdapter -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] (39) InputIteratorTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] (40) SortExecTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] -Arguments: [supplier_cnt#14 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST], true, 0 +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: [supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST], true, 0 (41) WholeStageCodegenTransformer (6) -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] Arguments: false (42) VeloxColumnarToRow -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt index 3130284e61d..fefc20cf743 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt @@ -47,12 +47,12 @@ Arguments: (isnotnull(l_partkey#1) AND isnotnull(l_quantity#2)) Output [3]: [p_partkey#4, p_brand#5, p_container#6] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/part] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#7), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#23), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct (4) FilterExecTransformer Input [3]: [p_partkey#4, p_brand#5, p_container#6] -Arguments: ((((isnotnull(p_brand#5) AND isnotnull(p_container#6)) AND (p_brand#5 = Brand#7)) AND (p_container#6 = MED BOX)) AND isnotnull(p_partkey#4)) +Arguments: ((((isnotnull(p_brand#5) AND isnotnull(p_container#6)) AND (p_brand#5 = Brand#23)) AND (p_container#6 = MED BOX)) AND isnotnull(p_partkey#4)) (5) ProjectExecTransformer Output [1]: [p_partkey#4] @@ -83,128 +83,128 @@ Output [3]: [l_quantity#2, l_extendedprice#3, p_partkey#4] Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, p_partkey#4] (12) FileSourceScanExecTransformer parquet spark_catalog.default.lineitem -Output [2]: [l_partkey#8, l_quantity#9] +Output [2]: [l_partkey#7, l_quantity#8] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/lineitem] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct (13) FilterExecTransformer -Input [2]: [l_partkey#8, l_quantity#9] -Arguments: isnotnull(l_partkey#8) +Input [2]: [l_partkey#7, l_quantity#8] +Arguments: isnotnull(l_partkey#7) (14) ProjectExecTransformer -Output [2]: [l_partkey#8, UnscaledValue(l_quantity#9) AS _pre_1#10] -Input [2]: [l_partkey#8, l_quantity#9] +Output [2]: [l_partkey#7, UnscaledValue(l_quantity#8) AS _pre_1#9] +Input [2]: [l_partkey#7, l_quantity#8] (15) FlushableHashAggregateExecTransformer -Input [2]: [l_partkey#8, _pre_1#10] -Keys [1]: [l_partkey#8] -Functions [1]: [partial_avg(_pre_1#10)] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [l_partkey#8, sum#13, count#14] +Input [2]: [l_partkey#7, _pre_1#9] +Keys [1]: [l_partkey#7] +Functions [1]: [partial_avg(_pre_1#9)] +Aggregate Attributes [2]: [sum#10, count#11] +Results [3]: [l_partkey#7, sum#12, count#13] (16) ProjectExecTransformer -Output [4]: [hash(l_partkey#8, 42) AS hash_partition_key#15, l_partkey#8, sum#13, count#14] -Input [3]: [l_partkey#8, sum#13, count#14] +Output [4]: [hash(l_partkey#7, 42) AS hash_partition_key#14, l_partkey#7, sum#12, count#13] +Input [3]: [l_partkey#7, sum#12, count#13] (17) WholeStageCodegenTransformer (2) -Input [4]: [hash_partition_key#15, l_partkey#8, sum#13, count#14] +Input [4]: [hash_partition_key#14, l_partkey#7, sum#12, count#13] Arguments: false (18) VeloxResizeBatches -Input [4]: [hash_partition_key#15, l_partkey#8, sum#13, count#14] +Input [4]: [hash_partition_key#14, l_partkey#7, sum#12, count#13] Arguments: 1024, 2147483647, 10485760 (19) ColumnarExchange -Input [4]: [hash_partition_key#15, l_partkey#8, sum#13, count#14] -Arguments: hashpartitioning(l_partkey#8, 1), ENSURE_REQUIREMENTS, [l_partkey#8, sum#13, count#14], [plan_id=2], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#14, l_partkey#7, sum#12, count#13] +Arguments: hashpartitioning(l_partkey#7, 1), ENSURE_REQUIREMENTS, [l_partkey#7, sum#12, count#13], [plan_id=2], [shuffle_writer_type=hash] (20) InputAdapter -Input [3]: [l_partkey#8, sum#13, count#14] +Input [3]: [l_partkey#7, sum#12, count#13] (21) InputIteratorTransformer -Input [3]: [l_partkey#8, sum#13, count#14] +Input [3]: [l_partkey#7, sum#12, count#13] (22) RegularHashAggregateExecTransformer -Input [3]: [l_partkey#8, sum#13, count#14] -Keys [1]: [l_partkey#8] -Functions [1]: [avg(UnscaledValue(l_quantity#9))] -Aggregate Attributes [1]: [avg(UnscaledValue(l_quantity#9))#16] -Results [2]: [l_partkey#8, avg(UnscaledValue(l_quantity#9))#16] +Input [3]: [l_partkey#7, sum#12, count#13] +Keys [1]: [l_partkey#7] +Functions [1]: [avg(UnscaledValue(l_quantity#8))] +Aggregate Attributes [1]: [avg(UnscaledValue(l_quantity#8))#15] +Results [2]: [l_partkey#7, avg(UnscaledValue(l_quantity#8))#15] (23) ProjectExecTransformer -Output [2]: [(0.2 * cast((avg(UnscaledValue(l_quantity#9))#16 / 1.0) as decimal(14,4))) AS (0.2 * avg(l_quantity))#17, l_partkey#8] -Input [2]: [l_partkey#8, avg(UnscaledValue(l_quantity#9))#16] +Output [2]: [(0.2 * cast((avg(UnscaledValue(l_quantity#8))#15 / 1.0) as decimal(14,4))) AS (0.2 * avg(l_quantity))#16, l_partkey#7] +Input [2]: [l_partkey#7, avg(UnscaledValue(l_quantity#8))#15] (24) FilterExecTransformer -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] -Arguments: isnotnull((0.2 * avg(l_quantity))#17) +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] +Arguments: isnotnull((0.2 * avg(l_quantity))#16) (25) WholeStageCodegenTransformer (3) -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] Arguments: false (26) ColumnarBroadcastExchange -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=3] (27) InputAdapter -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] (28) InputIteratorTransformer -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] (29) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#4] -Right keys [1]: [l_partkey#8] +Right keys [1]: [l_partkey#7] Join type: Inner -Join condition: (cast(l_quantity#2 as decimal(16,5)) < (0.2 * avg(l_quantity))#17) +Join condition: (cast(l_quantity#2 as decimal(16,5)) < (0.2 * avg(l_quantity))#16) (30) ProjectExecTransformer Output [1]: [l_extendedprice#3] -Input [5]: [l_quantity#2, l_extendedprice#3, p_partkey#4, (0.2 * avg(l_quantity))#17, l_partkey#8] +Input [5]: [l_quantity#2, l_extendedprice#3, p_partkey#4, (0.2 * avg(l_quantity))#16, l_partkey#7] (31) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#3] Keys: [] Functions [1]: [partial_sum(l_extendedprice#3)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [2]: [sum#20, isEmpty#21] +Aggregate Attributes [2]: [sum#17, isEmpty#18] +Results [2]: [sum#19, isEmpty#20] (32) WholeStageCodegenTransformer (4) -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: false (33) VeloxResizeBatches -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: 1024, 2147483647, 10485760 (34) ColumnarExchange -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4], [shuffle_writer_type=hash] (35) InputAdapter -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] (36) InputIteratorTransformer -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] (37) RegularHashAggregateExecTransformer -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Keys: [] Functions [1]: [sum(l_extendedprice#3)] -Aggregate Attributes [1]: [sum(l_extendedprice#3)#22] -Results [1]: [sum(l_extendedprice#3)#22] +Aggregate Attributes [1]: [sum(l_extendedprice#3)#21] +Results [1]: [sum(l_extendedprice#3)#21] (38) ProjectExecTransformer -Output [1]: [(sum(l_extendedprice#3)#22 / 7.0) AS avg_yearly#23] -Input [1]: [sum(l_extendedprice#3)#22] +Output [1]: [(sum(l_extendedprice#3)#21 / 7.0) AS avg_yearly#22] +Input [1]: [sum(l_extendedprice#3)#21] (39) WholeStageCodegenTransformer (5) -Input [1]: [avg_yearly#23] +Input [1]: [avg_yearly#22] Arguments: false (40) VeloxColumnarToRow -Input [1]: [avg_yearly#23] +Input [1]: [avg_yearly#22] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt index f2bb8c87b15..1e7568b06e2 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt @@ -35,12 +35,12 @@ Input [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipin Output [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/part] -PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#11),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#12),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#13),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#12),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#23),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#34),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct (5) FilterExecTransformer Input [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] -Arguments: (((isnotnull(p_size#9) AND (p_size#9 >= 1)) AND isnotnull(p_partkey#7)) AND (((((p_brand#8 = Brand#11) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#9 <= 5)) OR (((p_brand#8 = Brand#12) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#9 <= 10))) OR (((p_brand#8 = Brand#13) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#9 <= 15)))) +Arguments: (((isnotnull(p_size#9) AND (p_size#9 >= 1)) AND isnotnull(p_partkey#7)) AND (((((p_brand#8 = Brand#12) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#9 <= 5)) OR (((p_brand#8 = Brand#23) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#9 <= 10))) OR (((p_brand#8 = Brand#34) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#9 <= 15)))) (6) WholeStageCodegenTransformer (1) Input [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] @@ -60,48 +60,48 @@ Input [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] Left keys [1]: [l_partkey#1] Right keys [1]: [p_partkey#7] Join type: Inner -Join condition: (((((((p_brand#8 = Brand#11) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#2 >= 1)) AND (l_quantity#2 <= 11)) AND (p_size#9 <= 5)) OR (((((p_brand#8 = Brand#12) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#2 >= 10)) AND (l_quantity#2 <= 20)) AND (p_size#9 <= 10))) OR (((((p_brand#8 = Brand#13) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#2 >= 20)) AND (l_quantity#2 <= 30)) AND (p_size#9 <= 15))) +Join condition: (((((((p_brand#8 = Brand#12) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#2 >= 1)) AND (l_quantity#2 <= 11)) AND (p_size#9 <= 5)) OR (((((p_brand#8 = Brand#23) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#2 >= 10)) AND (l_quantity#2 <= 20)) AND (p_size#9 <= 10))) OR (((((p_brand#8 = Brand#34) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#2 >= 20)) AND (l_quantity#2 <= 30)) AND (p_size#9 <= 15))) (11) ProjectExecTransformer -Output [1]: [(l_extendedprice#3 * (1 - l_discount#4)) AS _pre_1#14] +Output [1]: [(l_extendedprice#3 * (1 - l_discount#4)) AS _pre_1#11] Input [8]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, p_partkey#7, p_brand#8, p_size#9, p_container#10] (12) FlushableHashAggregateExecTransformer -Input [1]: [_pre_1#14] +Input [1]: [_pre_1#11] Keys: [] -Functions [1]: [partial_sum(_pre_1#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [2]: [sum#17, isEmpty#18] +Functions [1]: [partial_sum(_pre_1#11)] +Aggregate Attributes [2]: [sum#12, isEmpty#13] +Results [2]: [sum#14, isEmpty#15] (13) WholeStageCodegenTransformer (2) -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Arguments: false (14) VeloxResizeBatches -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Arguments: 1024, 2147483647, 10485760 (15) ColumnarExchange -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2], [shuffle_writer_type=hash] (16) InputAdapter -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] (17) InputIteratorTransformer -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] (18) RegularHashAggregateExecTransformer -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Keys: [] Functions [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))] -Aggregate Attributes [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#19] -Results [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#19 AS revenue#20] +Aggregate Attributes [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#16] +Results [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#16 AS revenue#17] (19) WholeStageCodegenTransformer (3) -Input [1]: [revenue#20] +Input [1]: [revenue#17] Arguments: false (20) VeloxColumnarToRow -Input [1]: [revenue#20] +Input [1]: [revenue#17] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt index adbbf1e3d53..be8d675f5e5 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manufact_id#5] Input [2]: [i_item_sk#1, i_manufact_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2451911), (ss_sold_date_sk#13 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2451911), (ss_sold_date_sk#9 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_qoy#17] +Output [3]: [i_manufact_id#5, d_qoy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_qoy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#17, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#13, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#13, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223]), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] -Arguments: (((d_month_seq#27 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#16 >= 2451911)) AND (d_date_sk#16 <= 2452275)) AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] +Arguments: (((d_month_seq#23 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#12 >= 2451911)) AND (d_date_sk#12 <= 2452275)) AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_qoy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt index 53331732291..d255ed77d9c 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2451911), (ss_sold_date_sk#13 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2451911), (ss_sold_date_sk#9 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#16] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12] +Input [6]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_qoy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Output [3]: [i_manufact_id#5, d_qoy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#16, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#12, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#12, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223]), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] -Arguments: (((d_month_seq#27 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#15 >= 2451911)) AND (d_date_sk#15 <= 2452275)) AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] +Arguments: (((d_month_seq#23 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#11 >= 2451911)) AND (d_date_sk#11 <= 2452275)) AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_qoy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt index 053564324b1..2fbbba0d40d 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manager_id#5] Input [2]: [i_item_sk#1, i_manager_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2452123), (ss_sold_date_sk#13 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2452123), (ss_sold_date_sk#9 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_moy#17] +Output [3]: [i_manager_id#5, d_moy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_moy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#17, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#13, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#13, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230]), GreaterThanOrEqual(d_date_sk,2452123), LessThanOrEqual(d_date_sk,2452487), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] -Arguments: (((d_month_seq#27 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#16 >= 2452123)) AND (d_date_sk#16 <= 2452487)) AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] +Arguments: (((d_month_seq#23 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#12 >= 2452123)) AND (d_date_sk#12 <= 2452487)) AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_moy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt index 7aa9ef20afc..d72b84d6b82 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2452123), (ss_sold_date_sk#13 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2452123), (ss_sold_date_sk#9 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#16] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12] +Input [6]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_moy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Output [3]: [i_manager_id#5, d_moy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#16, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#12, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#12, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230]), GreaterThanOrEqual(d_date_sk,2452123), LessThanOrEqual(d_date_sk,2452487), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] -Arguments: (((d_month_seq#27 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#15 >= 2452123)) AND (d_date_sk#15 <= 2452487)) AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] +Arguments: (((d_month_seq#23 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#11 >= 2452123)) AND (d_date_sk#11 <= 2452487)) AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_moy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt index 1143d852955..71dd7e642ed 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manufact_id#5] Input [2]: [i_item_sk#1, i_manufact_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_qoy#17] +Output [3]: [i_manufact_id#5, d_qoy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_qoy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#17, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#13, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#13, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_qoy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt index 0289930482e..37ef088e869 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#16] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12] +Input [6]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_qoy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Output [3]: [i_manufact_id#5, d_qoy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#16, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#12, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#12, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_qoy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt index 2f2dc00832b..6e01c4ba841 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manager_id#5] Input [2]: [i_item_sk#1, i_manager_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_moy#17] +Output [3]: [i_manager_id#5, d_moy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_moy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#17, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#13, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#13, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_moy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt index c06e75c9dac..dc6374e1c70 100644 --- a/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt +++ b/gluten-ut/spark40/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#16] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12] +Input [6]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_moy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Output [3]: [i_manager_id#5, d_moy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#16, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#12, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#12, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_moy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenNormalizeIdsSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenNormalizeIdsSuite.scala new file mode 100644 index 00000000000..91875829c8d --- /dev/null +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenNormalizeIdsSuite.scala @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.SparkFunSuite + +/** + * Tests for the ExprId normalization used by the Gluten plan stability suites, in particular that + * string constants containing an ExprId-like fragment (e.g. Brand#12 in TPCH q19, or + * "scholaramalgamalg #14" in TPCDS q53) are not treated as ExprIds. See GLUTEN-12375. + */ +class GlutenNormalizeIdsSuite extends SparkFunSuite { + import GlutenPlanStabilityTestTrait._ + + test("ExprIds are normalized in encounter order") { + val plan = "Project [l_partkey#785, l_quantity#789L AS qty#801, l_partkey#785]" + assert(glutenNormalizeIds(plan) === "Project [l_partkey#1, l_quantity#2 AS qty#3, l_partkey#1]") + } + + test("plan ids and _pre_ names are normalized independently of ExprIds") { + val plan = "Exchange hashpartitioning(a#100, 200), [plan_id=1889]\n" + + "ReusedExchange [id=#1889]\n" + + "Project [split(c#101, ,, -1) AS _pre_7#102]" + val expected = "Exchange hashpartitioning(a#1, 200), [plan_id=1]\n" + + "ReusedExchange [id=#2]\n" + + "Project [split(c#2, ,, -1) AS _pre_1#3]" + assert(glutenNormalizeIds(plan) === expected) + } + + test("getHashLiterals extracts distinct hash-containing literals, longest first") { + val query = "select * from t where a = 'Brand#1' or a = 'Brand#12' " + + "or a = 'Brand#1' or b = 'no hash here' or c = 'scholaramalgamalg #14'" + assert(getHashLiterals(query) === Seq("scholaramalgamalg #14", "Brand#12", "Brand#1")) + } + + test("string literals with ExprId-like fragments are not normalized") { + val query = "select * from part where p_brand = 'Brand#12'" + val literals = getHashLiterals(query) + assert(literals === Seq("Brand#12")) + + val plan = "Filter (p_brand#785 = Brand#12)" + assert(glutenNormalizeIds(plan, literals) === "Filter (p_brand#1 = Brand#12)") + } + + test("literals padded by CHAR-type columns keep their value") { + // CHAR(50) columns pad literals with trailing spaces in explain output; the literal from the + // query text is still a prefix of the padded value. + val query = "select * from item where i_brand = 'scholaramalgamalg #14'" + val plan = "In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ]), i_brand#42" + val normalized = glutenNormalizeIds(plan, getHashLiterals(query)) + assert( + normalized === "In(i_brand, [exportiunivamalg #1 ,scholaramalgamalg #14 ]), i_brand#2") + } + + test("normalization is stable across different ExprId allocations") { + // The same plan printed in two JVM sessions: one where the ExprId counter is small enough to + // collide with the literal Brand#12 (an isolated suite run), and one where it is not (the + // whole module running in a single JVM). + val query = "select * from part where p_brand = 'Brand#12'" + val literals = getHashLiterals(query) + def plan(brandId: Int, sizeId: Int): String = + s"Filter ((p_brand#$brandId = Brand#12) AND (p_size#$sizeId > 1))" + + // Without protection, the literal token "#12" merges with the ExprId of p_brand#12 and the + // two sessions normalize differently. + assert(glutenNormalizeIds(plan(12, 13)) !== glutenNormalizeIds(plan(785, 786))) + + val isolated = glutenNormalizeIds(plan(12, 13), literals) + val sharedJvm = glutenNormalizeIds(plan(785, 786), literals) + assert(isolated === sharedJvm) + assert(isolated === "Filter ((p_brand#1 = Brand#12) AND (p_size#2 > 1))") + } +} diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala index c3187dbbab5..d4d8ea5f259 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala @@ -64,17 +64,17 @@ import scala.collection.mutable * For Spark 4.0, replace spark-4.1 with spark-4.0, spark41 with spark40, and SPARK_HOME * accordingly. * - * Note: Running all suites together in one JVM is recommended to avoid ExprId normalization issues - * where string constants (e.g., Brand#23 in TPCH q19) may collide with ExprId numbers. + * Note: String constants that contain an ExprId-like fragment (e.g., Brand#23 in TPCH q19) are + * shielded from ExprId normalization, so golden files are stable no matter whether the suites run + * together in one JVM or in isolation. */ trait GlutenPlanStabilityTestTrait { self: PlanStabilitySuite => + import GlutenPlanStabilityTestTrait._ + private val referenceRegex = "#\\d+".r - private val exprIdRegexp = "(?(?(plan_id=|id=#))\\d+".r private val preExprRegex = "_pre_\\d+".r - private val preExprIdRegex = "(?_pre_)\\d+".r private val clsName = this.getClass.getCanonicalName private lazy val glutenGoldenFilePath: String = { @@ -231,25 +231,6 @@ trait GlutenPlanStabilityTestTrait { simplifyNode(plan, 0) } - private def glutenNormalizeIds(plan: String): String = { - val exprIdMap = new mutable.HashMap[String, String]() - val exprIdNormalized = exprIdRegexp.replaceAllIn( - plan, - m => exprIdMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${exprIdMap.size + 1}")) - - val planIdMap = new mutable.HashMap[String, String]() - val planIdNormalized = planIdRegex.replaceAllIn( - exprIdNormalized, - m => planIdMap.getOrElseUpdate(s"$m", s"${m.group("prefix")}${planIdMap.size + 1}")) - - // Normalize _pre_N suffixes generated by Gluten's pre-projection optimization. - // These internal expression names vary depending on session state. - val preExprMap = new mutable.HashMap[String, String]() - preExprIdRegex.replaceAllIn( - planIdNormalized, - m => preExprMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${preExprMap.size + 1}")) - } - private def glutenNormalizeLocation(plan: String): String = { // Replace absolute paths in Location lines while preserving table names. // Handles both InMemoryFileIndex and CatalogFileIndex. @@ -271,7 +252,8 @@ trait GlutenPlanStabilityTestTrait { ) { val qe = sql(queryString).queryExecution val plan = qe.executedPlan - val explain = glutenNormalizeLocation(glutenNormalizeIds(qe.explainString(FormattedMode))) + val explain = glutenNormalizeLocation( + glutenNormalizeIds(qe.explainString(FormattedMode), getHashLiterals(queryString))) assert( ValidateRequirements.validate(plan), @@ -286,6 +268,62 @@ trait GlutenPlanStabilityTestTrait { } } +private[sql] object GlutenPlanStabilityTestTrait { + private val exprIdRegexp = "(?(?(plan_id=|id=#))\\d+".r + private val preExprIdRegex = "(?_pre_)\\d+".r + // Matches single-quoted SQL string literals containing an ExprId-like "#" fragment, + // e.g. 'Brand#12' in TPCH q19 or 'scholaramalgamalg #14' in TPCDS q53. + private val hashLiteralRegex = "'([^']*#\\d+[^']*)'".r + // Used to temporarily mask '#' inside such literals; never occurs in explain output. + private val hashMask = '\u0000' + + /** + * Extracts string literals that contain an ExprId-like "#" fragment from the query text. + * Such literals appear verbatim in the explain output, where they are syntactically + * indistinguishable from attribute ExprIds. Longer literals are returned first so that a literal + * that is a prefix of another one does not shadow it during masking. + */ + def getHashLiterals(queryString: String): Seq[String] = { + hashLiteralRegex.findAllMatchIn(queryString).map(_.group(1)).toSeq.distinct.sortBy(-_.length) + } + + /** + * Normalizes ExprIds, plan ids and Gluten's _pre_N expression names to sequential numbers in + * encounter order, so that plans can be compared against golden files across JVM sessions. + * + * Occurrences of `hashLiterals` (string constants from the query, e.g. Brand#12 in TPCH q19) are + * shielded from ExprId normalization. Without this, a literal like Brand#12 shares the "#12" + * token with whichever attribute happens to be allocated ExprId 12, so the normalized output + * would depend on the JVM's ExprId counter -- stable when a whole module runs in one JVM, but + * different in an isolated suite run. + */ + def glutenNormalizeIds(plan: String, hashLiterals: Seq[String] = Nil): String = { + val maskedPlan = hashLiterals.foldLeft(plan) { + (p, literal) => p.replace(literal, literal.replace('#', hashMask)) + } + + val exprIdMap = new mutable.HashMap[String, String]() + val exprIdNormalized = exprIdRegexp.replaceAllIn( + maskedPlan, + m => exprIdMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${exprIdMap.size + 1}")) + + val planIdMap = new mutable.HashMap[String, String]() + val planIdNormalized = planIdRegex.replaceAllIn( + exprIdNormalized, + m => planIdMap.getOrElseUpdate(s"$m", s"${m.group("prefix")}${planIdMap.size + 1}")) + + // Normalize _pre_N suffixes generated by Gluten's pre-projection optimization. + // These internal expression names vary depending on session state. + val preExprMap = new mutable.HashMap[String, String]() + val preExprNormalized = preExprIdRegex.replaceAllIn( + planIdNormalized, + m => preExprMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${preExprMap.size + 1}")) + + preExprNormalized.replace(hashMask, '#') + } +} + class GlutenTPCDSV1_4_PlanStabilitySuite extends TPCDSV1_4_PlanStabilitySuite with GlutenSQLTestsBaseTrait diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt index 261c2061e06..5b3793447bf 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q16/explain.txt @@ -82,12 +82,12 @@ Join condition: None Output [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/part] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#9)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_type), Not(EqualTo(p_brand,Brand#45)), Not(StringStartsWith(p_type,MEDIUM POLISHED)), In(p_size, [14,19,23,3,36,45,49,9]), IsNotNull(p_partkey)] ReadSchema: struct (12) FilterExecTransformer Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] -Arguments: (((((isnotnull(p_brand#6) AND isnotnull(p_type#7)) AND NOT (p_brand#6 = Brand#9)) AND NOT StartsWith(p_type#7, MEDIUM POLISHED)) AND p_size#8 IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#5)) +Arguments: (((((isnotnull(p_brand#6) AND isnotnull(p_type#7)) AND NOT (p_brand#6 = Brand#45)) AND NOT StartsWith(p_type#7, MEDIUM POLISHED)) AND p_size#8 IN (49,14,23,45,19,3,36,9)) AND isnotnull(p_partkey#5)) (13) WholeStageCodegenTransformer (2) Input [4]: [p_partkey#5, p_brand#6, p_type#7, p_size#8] @@ -121,19 +121,19 @@ Aggregate Attributes: [] Results [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] (20) ProjectExecTransformer -Output [5]: [hash(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 42) AS hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Output [5]: [hash(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 42) AS hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] (21) WholeStageCodegenTransformer (3) -Input [5]: [hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Arguments: false (22) VeloxResizeBatches -Input [5]: [hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Arguments: 1024, 2147483647, 10485760 (23) ColumnarExchange -Input [5]: [hash_partition_key#10, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] +Input [5]: [hash_partition_key#9, p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, ps_suppkey#2, 1), ENSURE_REQUIREMENTS, [p_brand#6, p_type#7, p_size#8, ps_suppkey#2], [plan_id=3], [shuffle_writer_type=hash] (24) InputAdapter @@ -153,64 +153,64 @@ Results [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Input [4]: [p_brand#6, p_type#7, p_size#8, ps_suppkey#2] Keys [3]: [p_brand#6, p_type#7, p_size#8] Functions [1]: [partial_count(distinct ps_suppkey#2)] -Aggregate Attributes [1]: [count(ps_suppkey#2)#11] -Results [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Aggregate Attributes [1]: [count(ps_suppkey#2)#10] +Results [4]: [p_brand#6, p_type#7, p_size#8, count#11] (28) ProjectExecTransformer -Output [5]: [hash(p_brand#6, p_type#7, p_size#8, 42) AS hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Output [5]: [hash(p_brand#6, p_type#7, p_size#8, 42) AS hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] (29) WholeStageCodegenTransformer (4) -Input [5]: [hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] +Input [5]: [hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] Arguments: false (30) VeloxResizeBatches -Input [5]: [hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] +Input [5]: [hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] Arguments: 1024, 2147483647, 10485760 (31) ColumnarExchange -Input [5]: [hash_partition_key#13, p_brand#6, p_type#7, p_size#8, count#12] -Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, 1), ENSURE_REQUIREMENTS, [p_brand#6, p_type#7, p_size#8, count#12], [plan_id=4], [shuffle_writer_type=hash] +Input [5]: [hash_partition_key#12, p_brand#6, p_type#7, p_size#8, count#11] +Arguments: hashpartitioning(p_brand#6, p_type#7, p_size#8, 1), ENSURE_REQUIREMENTS, [p_brand#6, p_type#7, p_size#8, count#11], [plan_id=4], [shuffle_writer_type=hash] (32) InputAdapter -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] (33) InputIteratorTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] (34) RegularHashAggregateExecTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, count#12] +Input [4]: [p_brand#6, p_type#7, p_size#8, count#11] Keys [3]: [p_brand#6, p_type#7, p_size#8] Functions [1]: [count(distinct ps_suppkey#2)] -Aggregate Attributes [1]: [count(ps_suppkey#2)#11] -Results [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#11 AS supplier_cnt#14] +Aggregate Attributes [1]: [count(ps_suppkey#2)#10] +Results [4]: [p_brand#6, p_type#7, p_size#8, count(ps_suppkey#2)#10 AS supplier_cnt#13] (35) WholeStageCodegenTransformer (5) -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] Arguments: false (36) VeloxResizeBatches -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] Arguments: 1024, 2147483647, 10485760 (37) ColumnarExchange -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] -Arguments: rangepartitioning(supplier_cnt#14 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=5], [shuffle_writer_type=hash] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: rangepartitioning(supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST, 1), ENSURE_REQUIREMENTS, [plan_id=5], [shuffle_writer_type=hash] (38) InputAdapter -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] (39) InputIteratorTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] (40) SortExecTransformer -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] -Arguments: [supplier_cnt#14 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST], true, 0 +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] +Arguments: [supplier_cnt#13 DESC NULLS LAST, p_brand#6 ASC NULLS FIRST, p_type#7 ASC NULLS FIRST, p_size#8 ASC NULLS FIRST], true, 0 (41) WholeStageCodegenTransformer (6) -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] Arguments: false (42) VeloxColumnarToRow -Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#14] +Input [4]: [p_brand#6, p_type#7, p_size#8, supplier_cnt#13] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt index 3130284e61d..fefc20cf743 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q17/explain.txt @@ -47,12 +47,12 @@ Arguments: (isnotnull(l_partkey#1) AND isnotnull(l_quantity#2)) Output [3]: [p_partkey#4, p_brand#5, p_container#6] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/part] -PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#7), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] +PushedFilters: [IsNotNull(p_brand), IsNotNull(p_container), EqualTo(p_brand,Brand#23), EqualTo(p_container,MED BOX), IsNotNull(p_partkey)] ReadSchema: struct (4) FilterExecTransformer Input [3]: [p_partkey#4, p_brand#5, p_container#6] -Arguments: ((((isnotnull(p_brand#5) AND isnotnull(p_container#6)) AND (p_brand#5 = Brand#7)) AND (p_container#6 = MED BOX)) AND isnotnull(p_partkey#4)) +Arguments: ((((isnotnull(p_brand#5) AND isnotnull(p_container#6)) AND (p_brand#5 = Brand#23)) AND (p_container#6 = MED BOX)) AND isnotnull(p_partkey#4)) (5) ProjectExecTransformer Output [1]: [p_partkey#4] @@ -83,128 +83,128 @@ Output [3]: [l_quantity#2, l_extendedprice#3, p_partkey#4] Input [4]: [l_partkey#1, l_quantity#2, l_extendedprice#3, p_partkey#4] (12) FileSourceScanExecTransformer parquet spark_catalog.default.lineitem -Output [2]: [l_partkey#8, l_quantity#9] +Output [2]: [l_partkey#7, l_quantity#8] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/lineitem] PushedFilters: [IsNotNull(l_partkey)] ReadSchema: struct (13) FilterExecTransformer -Input [2]: [l_partkey#8, l_quantity#9] -Arguments: isnotnull(l_partkey#8) +Input [2]: [l_partkey#7, l_quantity#8] +Arguments: isnotnull(l_partkey#7) (14) ProjectExecTransformer -Output [2]: [l_partkey#8, UnscaledValue(l_quantity#9) AS _pre_1#10] -Input [2]: [l_partkey#8, l_quantity#9] +Output [2]: [l_partkey#7, UnscaledValue(l_quantity#8) AS _pre_1#9] +Input [2]: [l_partkey#7, l_quantity#8] (15) FlushableHashAggregateExecTransformer -Input [2]: [l_partkey#8, _pre_1#10] -Keys [1]: [l_partkey#8] -Functions [1]: [partial_avg(_pre_1#10)] -Aggregate Attributes [2]: [sum#11, count#12] -Results [3]: [l_partkey#8, sum#13, count#14] +Input [2]: [l_partkey#7, _pre_1#9] +Keys [1]: [l_partkey#7] +Functions [1]: [partial_avg(_pre_1#9)] +Aggregate Attributes [2]: [sum#10, count#11] +Results [3]: [l_partkey#7, sum#12, count#13] (16) ProjectExecTransformer -Output [4]: [hash(l_partkey#8, 42) AS hash_partition_key#15, l_partkey#8, sum#13, count#14] -Input [3]: [l_partkey#8, sum#13, count#14] +Output [4]: [hash(l_partkey#7, 42) AS hash_partition_key#14, l_partkey#7, sum#12, count#13] +Input [3]: [l_partkey#7, sum#12, count#13] (17) WholeStageCodegenTransformer (2) -Input [4]: [hash_partition_key#15, l_partkey#8, sum#13, count#14] +Input [4]: [hash_partition_key#14, l_partkey#7, sum#12, count#13] Arguments: false (18) VeloxResizeBatches -Input [4]: [hash_partition_key#15, l_partkey#8, sum#13, count#14] +Input [4]: [hash_partition_key#14, l_partkey#7, sum#12, count#13] Arguments: 1024, 2147483647, 10485760 (19) ColumnarExchange -Input [4]: [hash_partition_key#15, l_partkey#8, sum#13, count#14] -Arguments: hashpartitioning(l_partkey#8, 1), ENSURE_REQUIREMENTS, [l_partkey#8, sum#13, count#14], [plan_id=2], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#14, l_partkey#7, sum#12, count#13] +Arguments: hashpartitioning(l_partkey#7, 1), ENSURE_REQUIREMENTS, [l_partkey#7, sum#12, count#13], [plan_id=2], [shuffle_writer_type=hash] (20) InputAdapter -Input [3]: [l_partkey#8, sum#13, count#14] +Input [3]: [l_partkey#7, sum#12, count#13] (21) InputIteratorTransformer -Input [3]: [l_partkey#8, sum#13, count#14] +Input [3]: [l_partkey#7, sum#12, count#13] (22) RegularHashAggregateExecTransformer -Input [3]: [l_partkey#8, sum#13, count#14] -Keys [1]: [l_partkey#8] -Functions [1]: [avg(UnscaledValue(l_quantity#9))] -Aggregate Attributes [1]: [avg(UnscaledValue(l_quantity#9))#16] -Results [2]: [l_partkey#8, avg(UnscaledValue(l_quantity#9))#16] +Input [3]: [l_partkey#7, sum#12, count#13] +Keys [1]: [l_partkey#7] +Functions [1]: [avg(UnscaledValue(l_quantity#8))] +Aggregate Attributes [1]: [avg(UnscaledValue(l_quantity#8))#15] +Results [2]: [l_partkey#7, avg(UnscaledValue(l_quantity#8))#15] (23) ProjectExecTransformer -Output [2]: [(0.2 * cast((avg(UnscaledValue(l_quantity#9))#16 / 1.0) as decimal(14,4))) AS (0.2 * avg(l_quantity))#17, l_partkey#8] -Input [2]: [l_partkey#8, avg(UnscaledValue(l_quantity#9))#16] +Output [2]: [(0.2 * cast((avg(UnscaledValue(l_quantity#8))#15 / 1.0) as decimal(14,4))) AS (0.2 * avg(l_quantity))#16, l_partkey#7] +Input [2]: [l_partkey#7, avg(UnscaledValue(l_quantity#8))#15] (24) FilterExecTransformer -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] -Arguments: isnotnull((0.2 * avg(l_quantity))#17) +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] +Arguments: isnotnull((0.2 * avg(l_quantity))#16) (25) WholeStageCodegenTransformer (3) -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] Arguments: false (26) ColumnarBroadcastExchange -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] Arguments: HashedRelationBroadcastMode(List(input[1, bigint, true]),false), [plan_id=3] (27) InputAdapter -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] (28) InputIteratorTransformer -Input [2]: [(0.2 * avg(l_quantity))#17, l_partkey#8] +Input [2]: [(0.2 * avg(l_quantity))#16, l_partkey#7] (29) BroadcastHashJoinExecTransformer Left keys [1]: [p_partkey#4] -Right keys [1]: [l_partkey#8] +Right keys [1]: [l_partkey#7] Join type: Inner -Join condition: (cast(l_quantity#2 as decimal(16,5)) < (0.2 * avg(l_quantity))#17) +Join condition: (cast(l_quantity#2 as decimal(16,5)) < (0.2 * avg(l_quantity))#16) (30) ProjectExecTransformer Output [1]: [l_extendedprice#3] -Input [5]: [l_quantity#2, l_extendedprice#3, p_partkey#4, (0.2 * avg(l_quantity))#17, l_partkey#8] +Input [5]: [l_quantity#2, l_extendedprice#3, p_partkey#4, (0.2 * avg(l_quantity))#16, l_partkey#7] (31) FlushableHashAggregateExecTransformer Input [1]: [l_extendedprice#3] Keys: [] Functions [1]: [partial_sum(l_extendedprice#3)] -Aggregate Attributes [2]: [sum#18, isEmpty#19] -Results [2]: [sum#20, isEmpty#21] +Aggregate Attributes [2]: [sum#17, isEmpty#18] +Results [2]: [sum#19, isEmpty#20] (32) WholeStageCodegenTransformer (4) -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: false (33) VeloxResizeBatches -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: 1024, 2147483647, 10485760 (34) ColumnarExchange -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=4], [shuffle_writer_type=hash] (35) InputAdapter -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] (36) InputIteratorTransformer -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] (37) RegularHashAggregateExecTransformer -Input [2]: [sum#20, isEmpty#21] +Input [2]: [sum#19, isEmpty#20] Keys: [] Functions [1]: [sum(l_extendedprice#3)] -Aggregate Attributes [1]: [sum(l_extendedprice#3)#22] -Results [1]: [sum(l_extendedprice#3)#22] +Aggregate Attributes [1]: [sum(l_extendedprice#3)#21] +Results [1]: [sum(l_extendedprice#3)#21] (38) ProjectExecTransformer -Output [1]: [(sum(l_extendedprice#3)#22 / 7.0) AS avg_yearly#23] -Input [1]: [sum(l_extendedprice#3)#22] +Output [1]: [(sum(l_extendedprice#3)#21 / 7.0) AS avg_yearly#22] +Input [1]: [sum(l_extendedprice#3)#21] (39) WholeStageCodegenTransformer (5) -Input [1]: [avg_yearly#23] +Input [1]: [avg_yearly#22] Arguments: false (40) VeloxColumnarToRow -Input [1]: [avg_yearly#23] +Input [1]: [avg_yearly#22] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt index f2bb8c87b15..1e7568b06e2 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/gluten-tpch-plan-stability/q19/explain.txt @@ -35,12 +35,12 @@ Input [6]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, l_shipin Output [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/part] -PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#11),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#12),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#13),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] +PushedFilters: [IsNotNull(p_size), GreaterThanOrEqual(p_size,1), IsNotNull(p_partkey), Or(Or(And(And(EqualTo(p_brand,Brand#12),In(p_container, [SM BOX,SM CASE,SM PACK,SM PKG])),LessThanOrEqual(p_size,5)),And(And(EqualTo(p_brand,Brand#23),In(p_container, [MED BAG,MED BOX,MED PACK,MED PKG])),LessThanOrEqual(p_size,10))),And(And(EqualTo(p_brand,Brand#34),In(p_container, [LG BOX,LG CASE,LG PACK,LG PKG])),LessThanOrEqual(p_size,15)))] ReadSchema: struct (5) FilterExecTransformer Input [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] -Arguments: (((isnotnull(p_size#9) AND (p_size#9 >= 1)) AND isnotnull(p_partkey#7)) AND (((((p_brand#8 = Brand#11) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#9 <= 5)) OR (((p_brand#8 = Brand#12) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#9 <= 10))) OR (((p_brand#8 = Brand#13) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#9 <= 15)))) +Arguments: (((isnotnull(p_size#9) AND (p_size#9 >= 1)) AND isnotnull(p_partkey#7)) AND (((((p_brand#8 = Brand#12) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (p_size#9 <= 5)) OR (((p_brand#8 = Brand#23) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (p_size#9 <= 10))) OR (((p_brand#8 = Brand#34) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (p_size#9 <= 15)))) (6) WholeStageCodegenTransformer (1) Input [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] @@ -60,48 +60,48 @@ Input [4]: [p_partkey#7, p_brand#8, p_size#9, p_container#10] Left keys [1]: [l_partkey#1] Right keys [1]: [p_partkey#7] Join type: Inner -Join condition: (((((((p_brand#8 = Brand#11) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#2 >= 1)) AND (l_quantity#2 <= 11)) AND (p_size#9 <= 5)) OR (((((p_brand#8 = Brand#12) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#2 >= 10)) AND (l_quantity#2 <= 20)) AND (p_size#9 <= 10))) OR (((((p_brand#8 = Brand#13) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#2 >= 20)) AND (l_quantity#2 <= 30)) AND (p_size#9 <= 15))) +Join condition: (((((((p_brand#8 = Brand#12) AND p_container#10 IN (SM CASE,SM BOX,SM PACK,SM PKG)) AND (l_quantity#2 >= 1)) AND (l_quantity#2 <= 11)) AND (p_size#9 <= 5)) OR (((((p_brand#8 = Brand#23) AND p_container#10 IN (MED BAG,MED BOX,MED PKG,MED PACK)) AND (l_quantity#2 >= 10)) AND (l_quantity#2 <= 20)) AND (p_size#9 <= 10))) OR (((((p_brand#8 = Brand#34) AND p_container#10 IN (LG CASE,LG BOX,LG PACK,LG PKG)) AND (l_quantity#2 >= 20)) AND (l_quantity#2 <= 30)) AND (p_size#9 <= 15))) (11) ProjectExecTransformer -Output [1]: [(l_extendedprice#3 * (1 - l_discount#4)) AS _pre_1#14] +Output [1]: [(l_extendedprice#3 * (1 - l_discount#4)) AS _pre_1#11] Input [8]: [l_partkey#1, l_quantity#2, l_extendedprice#3, l_discount#4, p_partkey#7, p_brand#8, p_size#9, p_container#10] (12) FlushableHashAggregateExecTransformer -Input [1]: [_pre_1#14] +Input [1]: [_pre_1#11] Keys: [] -Functions [1]: [partial_sum(_pre_1#14)] -Aggregate Attributes [2]: [sum#15, isEmpty#16] -Results [2]: [sum#17, isEmpty#18] +Functions [1]: [partial_sum(_pre_1#11)] +Aggregate Attributes [2]: [sum#12, isEmpty#13] +Results [2]: [sum#14, isEmpty#15] (13) WholeStageCodegenTransformer (2) -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Arguments: false (14) VeloxResizeBatches -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Arguments: 1024, 2147483647, 10485760 (15) ColumnarExchange -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Arguments: SinglePartition, ENSURE_REQUIREMENTS, [plan_id=2], [shuffle_writer_type=hash] (16) InputAdapter -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] (17) InputIteratorTransformer -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] (18) RegularHashAggregateExecTransformer -Input [2]: [sum#17, isEmpty#18] +Input [2]: [sum#14, isEmpty#15] Keys: [] Functions [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))] -Aggregate Attributes [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#19] -Results [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#19 AS revenue#20] +Aggregate Attributes [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#16] +Results [1]: [sum((l_extendedprice#3 * (1 - l_discount#4)))#16 AS revenue#17] (19) WholeStageCodegenTransformer (3) -Input [1]: [revenue#20] +Input [1]: [revenue#17] Arguments: false (20) VeloxColumnarToRow -Input [1]: [revenue#20] +Input [1]: [revenue#17] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt index adbbf1e3d53..be8d675f5e5 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manufact_id#5] Input [2]: [i_item_sk#1, i_manufact_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2451911), (ss_sold_date_sk#13 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2451911), (ss_sold_date_sk#9 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_qoy#17] +Output [3]: [i_manufact_id#5, d_qoy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_qoy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#17, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#13, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#13, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223]), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] -Arguments: (((d_month_seq#27 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#16 >= 2451911)) AND (d_date_sk#16 <= 2452275)) AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] +Arguments: (((d_month_seq#23 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#12 >= 2451911)) AND (d_date_sk#12 <= 2452275)) AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_qoy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt index 53331732291..d255ed77d9c 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q53/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2451911), (ss_sold_date_sk#13 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2451911), (ss_sold_date_sk#9 <= 2452275), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#16] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12] +Input [6]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_qoy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Output [3]: [i_manufact_id#5, d_qoy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#16, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#12, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#12, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1212,1213,1214,1215,1216,1217,1218,1219,1220,1221,1222,1223]), GreaterThanOrEqual(d_date_sk,2451911), LessThanOrEqual(d_date_sk,2452275), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] -Arguments: (((d_month_seq#27 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#15 >= 2451911)) AND (d_date_sk#15 <= 2452275)) AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] +Arguments: (((d_month_seq#23 INSET 1212, 1213, 1214, 1215, 1216, 1217, 1218, 1219, 1220, 1221, 1222, 1223 AND (d_date_sk#11 >= 2451911)) AND (d_date_sk#11 <= 2452275)) AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_qoy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt index 053564324b1..2fbbba0d40d 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manager_id#5] Input [2]: [i_item_sk#1, i_manager_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2452123), (ss_sold_date_sk#13 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2452123), (ss_sold_date_sk#9 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_moy#17] +Output [3]: [i_manager_id#5, d_moy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_moy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#17, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#13, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#13, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230]), GreaterThanOrEqual(d_date_sk,2452123), LessThanOrEqual(d_date_sk,2452487), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] -Arguments: (((d_month_seq#27 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#16 >= 2452123)) AND (d_date_sk#16 <= 2452487)) AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] +Arguments: (((d_month_seq#23 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#12 >= 2452123)) AND (d_date_sk#12 <= 2452487)) AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_moy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt index 7aa9ef20afc..d72b84d6b82 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-modified/q63/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), (ss_sold_date_sk#13 >= 2452123), (ss_sold_date_sk#13 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), (ss_sold_date_sk#9 >= 2452123), (ss_sold_date_sk#9 <= 2452487), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#16] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12] +Input [6]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_moy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Output [3]: [i_manager_id#5, d_moy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#16, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#12, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#12, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1219,1220,1221,1222,1223,1224,1225,1226,1227,1228,1229,1230]), GreaterThanOrEqual(d_date_sk,2452123), LessThanOrEqual(d_date_sk,2452487), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] -Arguments: (((d_month_seq#27 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#15 >= 2452123)) AND (d_date_sk#15 <= 2452487)) AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] +Arguments: (((d_month_seq#23 INSET 1219, 1220, 1221, 1222, 1223, 1224, 1225, 1226, 1227, 1228, 1229, 1230 AND (d_date_sk#11 >= 2452123)) AND (d_date_sk#11 <= 2452487)) AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_moy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt index 1143d852955..71dd7e642ed 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manufact_id#5] Input [2]: [i_item_sk#1, i_manufact_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_qoy#17] +Output [3]: [i_manufact_id#5, d_qoy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_qoy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#17, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#13, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#17, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#17, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#13, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#13, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#17, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#13, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_qoy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_qoy#17] +Output [2]: [d_date_sk#12, d_qoy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_qoy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_qoy#17] +Input [2]: [d_date_sk#12, d_qoy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt index 0289930482e..37ef088e869 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q53/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,reference ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,reference ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manufact_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manufact_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manufact_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16] -Input [6]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_qoy#16] +Output [4]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12] +Input [6]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_qoy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manufact_id#5, d_qoy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manufact_id#5, ss_store_sk#11, ss_sales_price#12, d_qoy#16, s_store_sk#17] +Output [3]: [i_manufact_id#5, d_qoy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manufact_id#5, ss_store_sk#7, ss_sales_price#8, d_qoy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, _pre_1#18] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, _pre_1#14] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manufact_id#5, d_qoy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, d_qoy#16, 42) AS hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Output [4]: [hash(i_manufact_id#5, d_qoy#12, 42) AS hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manufact_id#5, d_qoy#16, sum#20] -Arguments: hashpartitioning(i_manufact_id#5, d_qoy#16, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manufact_id#5, d_qoy#12, sum#16] +Arguments: hashpartitioning(i_manufact_id#5, d_qoy#12, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, d_qoy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manufact_id#5, d_qoy#16, sum#20] -Keys [2]: [i_manufact_id#5, d_qoy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manufact_id#5, d_qoy#12, sum#16] +Keys [2]: [i_manufact_id#5, d_qoy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#23, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manufact_id#5, d_qoy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manufact_id#5, 42) AS hash_partition_key#19, i_manufact_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manufact_id#5, d_qoy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manufact_id#5, 1), ENSURE_REQUIREMENTS, [i_manufact_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] Arguments: [i_manufact_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#26], [i_manufact_id#5] +Input [3]: [i_manufact_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manufact_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_quarterly_sales#22], [i_manufact_id#5] (41) FilterExecTransformer -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] -Arguments: CASE WHEN (avg_quarterly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_quarterly_sales#26)) / avg_quarterly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] +Arguments: CASE WHEN (avg_quarterly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_quarterly_sales#22)) / avg_quarterly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Input [4]: [i_manufact_id#5, sum_sales#24, _w0#25, avg_quarterly_sales#26] +Output [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Input [4]: [i_manufact_id#5, sum_sales#20, _w0#21, avg_quarterly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] -Arguments: 100, [avg_quarterly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26], 0 +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] +Arguments: 100, [avg_quarterly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST, i_manufact_id#5 ASC NULLS FIRST], [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manufact_id#5, sum_sales#24, avg_quarterly_sales#26] +Input [3]: [i_manufact_id#5, sum_sales#20, avg_quarterly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_qoy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_qoy#16] +Output [2]: [d_date_sk#11, d_qoy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_qoy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_qoy#16] +Input [2]: [d_date_sk#11, d_qoy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt index 2f2dc00832b..6e01c4ba841 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63.sf100/explain.txt @@ -40,12 +40,12 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] @@ -66,169 +66,169 @@ Input [2]: [i_item_sk#1, i_manager_id#5] Input [2]: [i_item_sk#1, i_manager_id#5] (8) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (9) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#15] +Output [1]: [s_store_sk#11] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (13) FilterExecTransformer -Input [1]: [s_store_sk#15] -Arguments: isnotnull(s_store_sk#15) +Input [1]: [s_store_sk#11] +Arguments: isnotnull(s_store_sk#11) (14) WholeStageCodegenTransformer (3) -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: false (15) ColumnarBroadcastExchange -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (16) InputAdapter -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (17) InputIteratorTransformer -Input [1]: [s_store_sk#15] +Input [1]: [s_store_sk#11] (18) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#15] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#11] Join type: Inner Join condition: None (19) ProjectExecTransformer -Output [3]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, s_store_sk#15] +Output [3]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, s_store_sk#11] (20) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#16, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] (21) InputAdapter -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (22) InputIteratorTransformer -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#16] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#12] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#17, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#16, d_moy#17] +Output [3]: [i_manager_id#5, d_moy#13, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#12, d_moy#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#13, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#17, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#13, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#17, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#17, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#17, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#13, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#13, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#13, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#17, sum#20] -Keys [2]: [i_manager_id#5, d_moy#17] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#13, sum#16] +Keys [2]: [i_manager_id#5, d_moy#13] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#17, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#13, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 8 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#16)) +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#12)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#16, d_moy#17] -Input [3]: [d_date_sk#16, d_month_seq#27, d_moy#17] +Output [2]: [d_date_sk#12, d_moy#13] +Input [3]: [d_date_sk#12, d_month_seq#23, d_moy#13] (49) WholeStageCodegenTransformer (2) -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#16, d_moy#17] +Input [2]: [d_date_sk#12, d_moy#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt index c06e75c9dac..dc6374e1c70 100644 --- a/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt +++ b/gluten-ut/spark41/src/test/resources/backends-velox/tpcds-plan-stability/gluten-approved-plans-v1_4/q63/explain.txt @@ -40,195 +40,195 @@ VeloxColumnarToRow (45) Output [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/item] -PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #6 ,scholaramalgamalg #7 ,scholaramalgamalg #8 ,scholaramalgamalg #6 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ]))), IsNotNull(i_item_sk)] +PushedFilters: [Or(And(And(In(i_category, [Books ,Children ,Electronics ]),In(i_class, [personal ,portable ,refernece ,self-help ])),In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ,scholaramalgamalg #7 ,scholaramalgamalg #9 ])),And(And(In(i_category, [Men ,Music ,Women ]),In(i_class, [accessories ,classical ,fragrances ,pants ])),In(i_brand, [amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ]))), IsNotNull(i_item_sk)] ReadSchema: struct (2) FilterExecTransformer Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] -Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #7 ,scholaramalgamalg #8 ,exportiunivamalg #6 ,scholaramalgamalg #6 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #9 ,edu packscholar #9 ,exportiimporto #9 ,importoamalg #9 ))) AND isnotnull(i_item_sk#1)) +Arguments: ((((i_category#4 IN (Books ,Children ,Electronics ) AND i_class#3 IN (personal ,portable ,refernece ,self-help )) AND i_brand#2 IN (scholaramalgamalg #14 ,scholaramalgamalg #7 ,exportiunivamalg #9 ,scholaramalgamalg #9 )) OR ((i_category#4 IN (Women ,Music ,Men ) AND i_class#3 IN (accessories ,classical ,fragrances ,pants )) AND i_brand#2 IN (amalgimporto #1 ,edu packscholar #1 ,exportiimporto #1 ,importoamalg #1 ))) AND isnotnull(i_item_sk#1)) (3) ProjectExecTransformer Output [2]: [i_item_sk#1, i_manager_id#5] Input [5]: [i_item_sk#1, i_brand#2, i_class#3, i_category#4, i_manager_id#5] (4) FileSourceScanExecTransformer parquet spark_catalog.default.store_sales -Output [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Batched: true Location: InMemoryFileIndex [] -PartitionFilters: [isnotnull(ss_sold_date_sk#13), dynamicpruningexpression(ss_sold_date_sk#13 IN dynamicpruning#14)] +PartitionFilters: [isnotnull(ss_sold_date_sk#9), dynamicpruningexpression(ss_sold_date_sk#9 IN dynamicpruning#10)] PushedFilters: [IsNotNull(ss_item_sk), IsNotNull(ss_store_sk)] ReadSchema: struct (5) FilterExecTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Arguments: (isnotnull(ss_item_sk#10) AND isnotnull(ss_store_sk#11)) +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Arguments: (isnotnull(ss_item_sk#6) AND isnotnull(ss_store_sk#7)) (6) WholeStageCodegenTransformer (2) -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: false (7) ColumnarBroadcastExchange -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=1] (8) InputAdapter -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (9) InputIteratorTransformer -Input [4]: [ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Input [4]: [ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (10) BroadcastHashJoinExecTransformer Left keys [1]: [i_item_sk#1] -Right keys [1]: [ss_item_sk#10] +Right keys [1]: [ss_item_sk#6] Join type: Inner Join condition: None (11) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] -Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#10, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] +Input [6]: [i_item_sk#1, i_manager_id#5, ss_item_sk#6, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9] (12) ReusedExchange [Reuses operator id: 50] -Output [2]: [d_date_sk#15, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] (13) InputAdapter -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (14) InputIteratorTransformer -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] (15) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_sold_date_sk#13] -Right keys [1]: [d_date_sk#15] +Left keys [1]: [ss_sold_date_sk#9] +Right keys [1]: [d_date_sk#11] Join type: Inner Join condition: None (16) ProjectExecTransformer -Output [4]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16] -Input [6]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, ss_sold_date_sk#13, d_date_sk#15, d_moy#16] +Output [4]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12] +Input [6]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, ss_sold_date_sk#9, d_date_sk#11, d_moy#12] (17) FileSourceScanExecTransformer parquet spark_catalog.default.store -Output [1]: [s_store_sk#17] +Output [1]: [s_store_sk#13] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/store] PushedFilters: [IsNotNull(s_store_sk)] ReadSchema: struct (18) FilterExecTransformer -Input [1]: [s_store_sk#17] -Arguments: isnotnull(s_store_sk#17) +Input [1]: [s_store_sk#13] +Arguments: isnotnull(s_store_sk#13) (19) WholeStageCodegenTransformer (4) -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: false (20) ColumnarBroadcastExchange -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, false] as bigint)),false), [plan_id=2] (21) InputAdapter -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (22) InputIteratorTransformer -Input [1]: [s_store_sk#17] +Input [1]: [s_store_sk#13] (23) BroadcastHashJoinExecTransformer -Left keys [1]: [ss_store_sk#11] -Right keys [1]: [s_store_sk#17] +Left keys [1]: [ss_store_sk#7] +Right keys [1]: [s_store_sk#13] Join type: Inner Join condition: None (24) ProjectExecTransformer -Output [3]: [i_manager_id#5, d_moy#16, UnscaledValue(ss_sales_price#12) AS _pre_1#18] -Input [5]: [i_manager_id#5, ss_store_sk#11, ss_sales_price#12, d_moy#16, s_store_sk#17] +Output [3]: [i_manager_id#5, d_moy#12, UnscaledValue(ss_sales_price#8) AS _pre_1#14] +Input [5]: [i_manager_id#5, ss_store_sk#7, ss_sales_price#8, d_moy#12, s_store_sk#13] (25) FlushableHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, _pre_1#18] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [partial_sum(_pre_1#18)] -Aggregate Attributes [1]: [sum#19] -Results [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, _pre_1#14] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [partial_sum(_pre_1#14)] +Aggregate Attributes [1]: [sum#15] +Results [3]: [i_manager_id#5, d_moy#12, sum#16] (26) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, d_moy#16, 42) AS hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Output [4]: [hash(i_manager_id#5, d_moy#12, 42) AS hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (27) WholeStageCodegenTransformer (5) -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: false (28) VeloxResizeBatches -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] Arguments: 1024, 2147483647, 10485760 (29) ColumnarExchange -Input [4]: [hash_partition_key#21, i_manager_id#5, d_moy#16, sum#20] -Arguments: hashpartitioning(i_manager_id#5, d_moy#16, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#16, sum#20], [plan_id=3], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#17, i_manager_id#5, d_moy#12, sum#16] +Arguments: hashpartitioning(i_manager_id#5, d_moy#12, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, d_moy#12, sum#16], [plan_id=3], [shuffle_writer_type=hash] (30) InputAdapter -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (31) InputIteratorTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] (32) RegularHashAggregateExecTransformer -Input [3]: [i_manager_id#5, d_moy#16, sum#20] -Keys [2]: [i_manager_id#5, d_moy#16] -Functions [1]: [sum(UnscaledValue(ss_sales_price#12))] -Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#12))#22] -Results [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Input [3]: [i_manager_id#5, d_moy#12, sum#16] +Keys [2]: [i_manager_id#5, d_moy#12] +Functions [1]: [sum(UnscaledValue(ss_sales_price#8))] +Aggregate Attributes [1]: [sum(UnscaledValue(ss_sales_price#8))#18] +Results [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (33) ProjectExecTransformer -Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#23, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS sum_sales#24, MakeDecimal(sum(UnscaledValue(ss_sales_price#12))#22,17,2) AS _w0#25] -Input [3]: [i_manager_id#5, d_moy#16, sum(UnscaledValue(ss_sales_price#12))#22] +Output [4]: [hash(i_manager_id#5, 42) AS hash_partition_key#19, i_manager_id#5, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS sum_sales#20, MakeDecimal(sum(UnscaledValue(ss_sales_price#8))#18,17,2) AS _w0#21] +Input [3]: [i_manager_id#5, d_moy#12, sum(UnscaledValue(ss_sales_price#8))#18] (34) WholeStageCodegenTransformer (6) -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: false (35) VeloxResizeBatches -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] Arguments: 1024, 2147483647, 10485760 (36) ColumnarExchange -Input [4]: [hash_partition_key#23, i_manager_id#5, sum_sales#24, _w0#25] -Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#24, _w0#25], [plan_id=4], [shuffle_writer_type=hash] +Input [4]: [hash_partition_key#19, i_manager_id#5, sum_sales#20, _w0#21] +Arguments: hashpartitioning(i_manager_id#5, 1), ENSURE_REQUIREMENTS, [i_manager_id#5, sum_sales#20, _w0#21], [plan_id=4], [shuffle_writer_type=hash] (37) InputAdapter -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (38) InputIteratorTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] (39) SortExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] Arguments: [i_manager_id#5 ASC NULLS FIRST], false, 0 (40) WindowExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, _w0#25] -Arguments: [avg(_w0#25) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#26], [i_manager_id#5] +Input [3]: [i_manager_id#5, sum_sales#20, _w0#21] +Arguments: [avg(_w0#21) windowspecdefinition(i_manager_id#5, specifiedwindowframe(RowFrame, unboundedpreceding$(), unboundedfollowing$())) AS avg_monthly_sales#22], [i_manager_id#5] (41) FilterExecTransformer -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] -Arguments: CASE WHEN (avg_monthly_sales#26 > 0.000000) THEN ((abs((sum_sales#24 - avg_monthly_sales#26)) / avg_monthly_sales#26) > 0.1000000000000000) ELSE false END +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] +Arguments: CASE WHEN (avg_monthly_sales#22 > 0.000000) THEN ((abs((sum_sales#20 - avg_monthly_sales#22)) / avg_monthly_sales#22) > 0.1000000000000000) ELSE false END (42) ProjectExecTransformer -Output [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Input [4]: [i_manager_id#5, sum_sales#24, _w0#25, avg_monthly_sales#26] +Output [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Input [4]: [i_manager_id#5, sum_sales#20, _w0#21, avg_monthly_sales#22] (43) WholeStageCodegenTransformer (7) -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] Arguments: false (44) TakeOrderedAndProjectExecTransformer -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] -Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#26 ASC NULLS FIRST, sum_sales#24 ASC NULLS FIRST], [i_manager_id#5, sum_sales#24, avg_monthly_sales#26], 0 +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] +Arguments: 100, [i_manager_id#5 ASC NULLS FIRST, avg_monthly_sales#22 ASC NULLS FIRST, sum_sales#20 ASC NULLS FIRST], [i_manager_id#5, sum_sales#20, avg_monthly_sales#22], 0 (45) VeloxColumnarToRow -Input [3]: [i_manager_id#5, sum_sales#24, avg_monthly_sales#26] +Input [3]: [i_manager_id#5, sum_sales#20, avg_monthly_sales#22] ===== Subqueries ===== -Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#13 IN dynamicpruning#14 +Subquery:1 Hosting operator id = 4 Hosting Expression = ss_sold_date_sk#9 IN dynamicpruning#10 ColumnarBroadcastExchange (50) +- ^ ProjectExecTransformer (48) +- ^ FilterExecTransformer (47) @@ -236,26 +236,26 @@ ColumnarBroadcastExchange (50) (46) FileSourceScanExecTransformer parquet spark_catalog.default.date_dim -Output [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] Batched: true Location: InMemoryFileIndex [{warehouse_dir}/date_dim] PushedFilters: [In(d_month_seq, [1200,1201,1202,1203,1204,1205,1206,1207,1208,1209,1210,1211]), IsNotNull(d_date_sk)] ReadSchema: struct (47) FilterExecTransformer -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] -Arguments: (d_month_seq#27 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#15)) +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] +Arguments: (d_month_seq#23 INSET 1200, 1201, 1202, 1203, 1204, 1205, 1206, 1207, 1208, 1209, 1210, 1211 AND isnotnull(d_date_sk#11)) (48) ProjectExecTransformer -Output [2]: [d_date_sk#15, d_moy#16] -Input [3]: [d_date_sk#15, d_month_seq#27, d_moy#16] +Output [2]: [d_date_sk#11, d_moy#12] +Input [3]: [d_date_sk#11, d_month_seq#23, d_moy#12] (49) WholeStageCodegenTransformer (1) -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: false (50) ColumnarBroadcastExchange -Input [2]: [d_date_sk#15, d_moy#16] +Input [2]: [d_date_sk#11, d_moy#12] Arguments: HashedRelationBroadcastMode(List(cast(input[0, int, true] as bigint)),false), [plan_id=5] diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenNormalizeIdsSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenNormalizeIdsSuite.scala new file mode 100644 index 00000000000..91875829c8d --- /dev/null +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenNormalizeIdsSuite.scala @@ -0,0 +1,87 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql + +import org.apache.spark.SparkFunSuite + +/** + * Tests for the ExprId normalization used by the Gluten plan stability suites, in particular that + * string constants containing an ExprId-like fragment (e.g. Brand#12 in TPCH q19, or + * "scholaramalgamalg #14" in TPCDS q53) are not treated as ExprIds. See GLUTEN-12375. + */ +class GlutenNormalizeIdsSuite extends SparkFunSuite { + import GlutenPlanStabilityTestTrait._ + + test("ExprIds are normalized in encounter order") { + val plan = "Project [l_partkey#785, l_quantity#789L AS qty#801, l_partkey#785]" + assert(glutenNormalizeIds(plan) === "Project [l_partkey#1, l_quantity#2 AS qty#3, l_partkey#1]") + } + + test("plan ids and _pre_ names are normalized independently of ExprIds") { + val plan = "Exchange hashpartitioning(a#100, 200), [plan_id=1889]\n" + + "ReusedExchange [id=#1889]\n" + + "Project [split(c#101, ,, -1) AS _pre_7#102]" + val expected = "Exchange hashpartitioning(a#1, 200), [plan_id=1]\n" + + "ReusedExchange [id=#2]\n" + + "Project [split(c#2, ,, -1) AS _pre_1#3]" + assert(glutenNormalizeIds(plan) === expected) + } + + test("getHashLiterals extracts distinct hash-containing literals, longest first") { + val query = "select * from t where a = 'Brand#1' or a = 'Brand#12' " + + "or a = 'Brand#1' or b = 'no hash here' or c = 'scholaramalgamalg #14'" + assert(getHashLiterals(query) === Seq("scholaramalgamalg #14", "Brand#12", "Brand#1")) + } + + test("string literals with ExprId-like fragments are not normalized") { + val query = "select * from part where p_brand = 'Brand#12'" + val literals = getHashLiterals(query) + assert(literals === Seq("Brand#12")) + + val plan = "Filter (p_brand#785 = Brand#12)" + assert(glutenNormalizeIds(plan, literals) === "Filter (p_brand#1 = Brand#12)") + } + + test("literals padded by CHAR-type columns keep their value") { + // CHAR(50) columns pad literals with trailing spaces in explain output; the literal from the + // query text is still a prefix of the padded value. + val query = "select * from item where i_brand = 'scholaramalgamalg #14'" + val plan = "In(i_brand, [exportiunivamalg #9 ,scholaramalgamalg #14 ]), i_brand#42" + val normalized = glutenNormalizeIds(plan, getHashLiterals(query)) + assert( + normalized === "In(i_brand, [exportiunivamalg #1 ,scholaramalgamalg #14 ]), i_brand#2") + } + + test("normalization is stable across different ExprId allocations") { + // The same plan printed in two JVM sessions: one where the ExprId counter is small enough to + // collide with the literal Brand#12 (an isolated suite run), and one where it is not (the + // whole module running in a single JVM). + val query = "select * from part where p_brand = 'Brand#12'" + val literals = getHashLiterals(query) + def plan(brandId: Int, sizeId: Int): String = + s"Filter ((p_brand#$brandId = Brand#12) AND (p_size#$sizeId > 1))" + + // Without protection, the literal token "#12" merges with the ExprId of p_brand#12 and the + // two sessions normalize differently. + assert(glutenNormalizeIds(plan(12, 13)) !== glutenNormalizeIds(plan(785, 786))) + + val isolated = glutenNormalizeIds(plan(12, 13), literals) + val sharedJvm = glutenNormalizeIds(plan(785, 786), literals) + assert(isolated === sharedJvm) + assert(isolated === "Filter ((p_brand#1 = Brand#12) AND (p_size#2 > 1))") + } +} diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala index c01599572b6..c3b4a51aa34 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/GlutenPlanStabilitySuite.scala @@ -64,17 +64,17 @@ import scala.collection.mutable * For Spark 4.0, replace spark-4.1 with spark-4.0, spark41 with spark40, and SPARK_HOME * accordingly. * - * Note: Running all suites together in one JVM is recommended to avoid ExprId normalization issues - * where string constants (e.g., Brand#23 in TPCH q19) may collide with ExprId numbers. + * Note: String constants that contain an ExprId-like fragment (e.g., Brand#23 in TPCH q19) are + * shielded from ExprId normalization, so golden files are stable no matter whether the suites run + * together in one JVM or in isolation. */ trait GlutenPlanStabilityTestTrait { self: PlanStabilitySuite => + import GlutenPlanStabilityTestTrait._ + private val referenceRegex = "#\\d+".r - private val exprIdRegexp = "(?(?(plan_id=|id=#))\\d+".r private val preExprRegex = "_pre_\\d+".r - private val preExprIdRegex = "(?_pre_)\\d+".r private val clsName = this.getClass.getCanonicalName private lazy val glutenGoldenFilePath: String = { @@ -231,25 +231,6 @@ trait GlutenPlanStabilityTestTrait { simplifyNode(plan, 0) } - private def glutenNormalizeIds(plan: String): String = { - val exprIdMap = new mutable.HashMap[String, String]() - val exprIdNormalized = exprIdRegexp.replaceAllIn( - plan, - m => exprIdMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${exprIdMap.size + 1}")) - - val planIdMap = new mutable.HashMap[String, String]() - val planIdNormalized = planIdRegex.replaceAllIn( - exprIdNormalized, - m => planIdMap.getOrElseUpdate(s"$m", s"${m.group("prefix")}${planIdMap.size + 1}")) - - // Normalize _pre_N suffixes generated by Gluten's pre-projection optimization. - // These internal expression names vary depending on session state. - val preExprMap = new mutable.HashMap[String, String]() - preExprIdRegex.replaceAllIn( - planIdNormalized, - m => preExprMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${preExprMap.size + 1}")) - } - private def glutenNormalizeLocation(plan: String): String = { // Replace absolute paths in Location lines while preserving table names. // Handles both InMemoryFileIndex and CatalogFileIndex. @@ -271,7 +252,8 @@ trait GlutenPlanStabilityTestTrait { ) { val qe = sql(queryString).queryExecution val plan = qe.executedPlan - val explain = glutenNormalizeLocation(glutenNormalizeIds(qe.explainString(FormattedMode))) + val explain = glutenNormalizeLocation( + glutenNormalizeIds(qe.explainString(FormattedMode), getHashLiterals(queryString))) assert( ValidateRequirements.validate(plan), @@ -286,6 +268,62 @@ trait GlutenPlanStabilityTestTrait { } } +private[sql] object GlutenPlanStabilityTestTrait { + private val exprIdRegexp = "(?(?(plan_id=|id=#))\\d+".r + private val preExprIdRegex = "(?_pre_)\\d+".r + // Matches single-quoted SQL string literals containing an ExprId-like "#" fragment, + // e.g. 'Brand#12' in TPCH q19 or 'scholaramalgamalg #14' in TPCDS q53. + private val hashLiteralRegex = "'([^']*#\\d+[^']*)'".r + // Used to temporarily mask '#' inside such literals; never occurs in explain output. + private val hashMask = '\u0000' + + /** + * Extracts string literals that contain an ExprId-like "#" fragment from the query text. + * Such literals appear verbatim in the explain output, where they are syntactically + * indistinguishable from attribute ExprIds. Longer literals are returned first so that a literal + * that is a prefix of another one does not shadow it during masking. + */ + def getHashLiterals(queryString: String): Seq[String] = { + hashLiteralRegex.findAllMatchIn(queryString).map(_.group(1)).toSeq.distinct.sortBy(-_.length) + } + + /** + * Normalizes ExprIds, plan ids and Gluten's _pre_N expression names to sequential numbers in + * encounter order, so that plans can be compared against golden files across JVM sessions. + * + * Occurrences of `hashLiterals` (string constants from the query, e.g. Brand#12 in TPCH q19) are + * shielded from ExprId normalization. Without this, a literal like Brand#12 shares the "#12" + * token with whichever attribute happens to be allocated ExprId 12, so the normalized output + * would depend on the JVM's ExprId counter -- stable when a whole module runs in one JVM, but + * different in an isolated suite run. + */ + def glutenNormalizeIds(plan: String, hashLiterals: Seq[String] = Nil): String = { + val maskedPlan = hashLiterals.foldLeft(plan) { + (p, literal) => p.replace(literal, literal.replace('#', hashMask)) + } + + val exprIdMap = new mutable.HashMap[String, String]() + val exprIdNormalized = exprIdRegexp.replaceAllIn( + maskedPlan, + m => exprIdMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${exprIdMap.size + 1}")) + + val planIdMap = new mutable.HashMap[String, String]() + val planIdNormalized = planIdRegex.replaceAllIn( + exprIdNormalized, + m => planIdMap.getOrElseUpdate(s"$m", s"${m.group("prefix")}${planIdMap.size + 1}")) + + // Normalize _pre_N suffixes generated by Gluten's pre-projection optimization. + // These internal expression names vary depending on session state. + val preExprMap = new mutable.HashMap[String, String]() + val preExprNormalized = preExprIdRegex.replaceAllIn( + planIdNormalized, + m => preExprMap.getOrElseUpdate(m.toString(), s"${m.group("prefix")}${preExprMap.size + 1}")) + + preExprNormalized.replace(hashMask, '#') + } +} + class GlutenTPCDSV1_4_PlanStabilitySuite extends TPCDSV1_4_PlanStabilitySuite with GlutenSQLTestsBaseTrait