diff --git a/mysql-test/main/win_streaming.result b/mysql-test/main/win_streaming.result new file mode 100644 index 0000000000000..cc9e197d4c639 --- /dev/null +++ b/mysql-test/main/win_streaming.result @@ -0,0 +1,503 @@ +CREATE TABLE t1 (pk INT PRIMARY KEY, a INT, b INT); +INSERT INTO t1 VALUES (1, 1, 10); +INSERT INTO t1 VALUES (2, 1, 10); +INSERT INTO t1 VALUES (3, 1, 20); +INSERT INTO t1 VALUES (4, 2, 20); +INSERT INTO t1 VALUES (5, 2, 20); +INSERT INTO t1 VALUES (6, 2, 30); +INSERT INTO t1 VALUES (7, 3, 10); +INSERT INTO t1 VALUES (8, 3, 30); +INSERT INTO t1 VALUES (9, 3, 30); +EXPLAIN EXTENDED SELECT pk, a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (PARTITION BY a ORDER BY b, pk); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +EXPLAIN EXTENDED SELECT SQL_BUFFER_RESULT pk, a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (PARTITION BY a ORDER BY b, pk); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +SELECT pk, a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (PARTITION BY a ORDER BY b, pk); +pk a b rn rnk drnk +1 1 10 1 1 1 +2 1 10 2 2 2 +3 1 20 3 3 3 +4 2 20 1 1 1 +5 2 20 2 2 2 +6 2 30 3 3 3 +7 3 10 1 1 1 +8 3 30 2 2 2 +9 3 30 3 3 3 +SELECT SQL_BUFFER_RESULT pk, a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (PARTITION BY a ORDER BY b, pk); +pk a b rn rnk drnk +1 1 10 1 1 1 +2 1 10 2 2 2 +3 1 20 3 3 3 +4 2 20 1 1 1 +5 2 20 2 2 2 +6 2 30 3 3 3 +7 3 10 1 1 1 +8 3 30 2 2 2 +9 3 30 3 3 3 +EXPLAIN EXTENDED SELECT pk, a, b, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +SELECT pk, a, b, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a); +pk a b rnk drnk +1 1 10 1 1 +2 1 10 1 1 +3 1 20 1 1 +4 2 20 4 2 +5 2 20 4 2 +6 2 30 4 2 +7 3 10 7 3 +8 3 30 7 3 +9 3 30 7 3 +SELECT SQL_BUFFER_RESULT pk, a, b, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a); +pk a b rnk drnk +1 1 10 1 1 +2 1 10 1 1 +3 1 20 1 1 +4 2 20 4 2 +5 2 20 4 2 +6 2 30 4 2 +7 3 10 7 3 +8 3 30 7 3 +9 3 30 7 3 +EXPLAIN EXTENDED SELECT pk, a, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a, pk); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +SELECT pk, a, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a, pk); +pk a rn rnk drnk +1 1 1 1 1 +2 1 2 2 2 +3 1 3 3 3 +4 2 4 4 4 +5 2 5 5 5 +6 2 6 6 6 +7 3 7 7 7 +8 3 8 8 8 +9 3 9 9 9 +SELECT SQL_BUFFER_RESULT pk, a, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a, pk); +pk a rn rnk drnk +1 1 1 1 1 +2 1 2 2 2 +3 1 3 3 3 +4 2 4 4 4 +5 2 5 5 5 +6 2 6 6 6 +7 3 7 7 7 +8 3 8 8 8 +9 3 9 9 9 +EXPLAIN FORMAT=JSON SELECT pk, a, b, rank() OVER (ORDER BY a, b) AS rnk FROM t1 ORDER BY a; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "cost": "COST_REPLACED", + "nested_loop": [ + { + "read_sorted_file": { + "filesort": { + "sort_key": "t1.a, t1.b", + "table": { + "table_name": "t1", + "access_type": "ALL", + "loops": 1, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100 + } + } + } + } + ] + } +} +SELECT pk, a, b, rank() OVER (ORDER BY a, b) AS rnk FROM t1 ORDER BY a; +pk a b rnk +1 1 10 1 +2 1 10 1 +3 1 20 3 +4 2 20 4 +5 2 20 4 +6 2 30 6 +7 3 10 7 +8 3 30 8 +9 3 30 8 +SELECT SQL_BUFFER_RESULT pk, a, b, rank() OVER (ORDER BY a, b) AS rnk FROM t1 ORDER BY a; +pk a b rnk +1 1 10 1 +2 1 10 1 +3 1 20 3 +4 2 20 4 +5 2 20 4 +6 2 30 6 +7 3 10 7 +8 3 30 8 +9 3 30 8 +EXPLAIN FORMAT=JSON SELECT pk, a, b, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a, b; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "cost": "COST_REPLACED", + "nested_loop": [ + { + "read_sorted_file": { + "filesort": { + "sort_key": "t1.a, t1.b", + "table": { + "table_name": "t1", + "access_type": "ALL", + "loops": 1, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100 + } + } + } + } + ] + } +} +SELECT pk, a, b, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a, b; +pk a b rnk +1 1 10 1 +2 1 10 1 +3 1 20 1 +4 2 20 4 +5 2 20 4 +6 2 30 4 +7 3 10 7 +8 3 30 7 +9 3 30 7 +SELECT SQL_BUFFER_RESULT pk, a, b, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a, b; +pk a b rnk +1 1 10 1 +2 1 10 1 +3 1 20 1 +4 2 20 4 +5 2 20 4 +6 2 30 4 +7 3 10 7 +8 3 30 7 +9 3 30 7 +EXPLAIN FORMAT=JSON SELECT pk, a, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a; +EXPLAIN +{ + "query_block": { + "select_id": 1, + "cost": "COST_REPLACED", + "nested_loop": [ + { + "read_sorted_file": { + "filesort": { + "sort_key": "t1.a", + "table": { + "table_name": "t1", + "access_type": "ALL", + "loops": 1, + "rows": 9, + "cost": "COST_REPLACED", + "filtered": 100 + } + } + } + } + ] + } +} +SELECT pk, a, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a; +pk a rnk +1 1 1 +2 1 1 +3 1 1 +4 2 4 +5 2 4 +6 2 4 +7 3 7 +8 3 7 +9 3 7 +SELECT SQL_BUFFER_RESULT pk, a, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a; +pk a rnk +1 1 1 +2 1 1 +3 1 1 +4 2 4 +5 2 4 +6 2 4 +7 3 7 +8 3 7 +9 3 7 +EXPLAIN EXTENDED SELECT pk, a, rank() OVER w AS r, rank() OVER w + 1 AS r_plus, rank() OVER w - dense_rank() OVER w AS diff FROM t1 WINDOW w AS (ORDER BY a); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +SELECT pk, a, rank() OVER w AS r, rank() OVER w + 1 AS r_plus, rank() OVER w - dense_rank() OVER w AS diff FROM t1 WINDOW w AS (ORDER BY a); +pk a r r_plus diff +1 1 1 2 0 +2 1 1 2 0 +3 1 1 2 0 +4 2 4 5 2 +5 2 4 5 2 +6 2 4 5 2 +7 3 7 8 4 +8 3 7 8 4 +9 3 7 8 4 +SELECT SQL_BUFFER_RESULT pk, a, rank() OVER w AS r, rank() OVER w + 1 AS r_plus, rank() OVER w - dense_rank() OVER w AS diff FROM t1 WINDOW w AS (ORDER BY a); +pk a r r_plus diff +1 1 1 2 0 +2 1 1 2 0 +3 1 1 2 0 +4 2 4 5 2 +5 2 4 5 2 +6 2 4 5 2 +7 3 7 8 4 +8 3 7 8 4 +9 3 7 8 4 +CREATE TABLE t2 (pk INT PRIMARY KEY, c INT); +INSERT INTO t2 VALUES (1, 100); +INSERT INTO t2 VALUES (2, 200); +INSERT INTO t2 VALUES (3, 300); +INSERT INTO t2 VALUES (4, 400); +INSERT INTO t2 VALUES (5, 500); +INSERT INTO t2 VALUES (6, 600); +INSERT INTO t2 VALUES (7, 700); +INSERT INTO t2 VALUES (8, 800); +INSERT INTO t2 VALUES (9, 900); +EXPLAIN EXTENDED SELECT t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 JOIN t2 ON t1.pk = t2.pk; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL PRIMARY NULL NULL NULL 9 100.00 Using filesort +1 SIMPLE t2 eq_ref PRIMARY PRIMARY 4 test.t1.pk 1 100.00 Using index +SELECT t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 JOIN t2 ON t1.pk = t2.pk; +pk a b rnk +1 1 10 1 +2 1 10 2 +3 1 20 4 +4 2 20 5 +5 2 20 6 +6 2 30 7 +7 3 10 3 +8 3 30 8 +9 3 30 9 +SELECT SQL_BUFFER_RESULT t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 JOIN t2 ON t1.pk = t2.pk; +pk a b rnk +1 1 10 1 +2 1 10 2 +3 1 20 4 +4 2 20 5 +5 2 20 6 +6 2 30 7 +7 3 10 3 +8 3 30 8 +9 3 30 9 +EXPLAIN EXTENDED SELECT t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 LEFT JOIN t2 ON t1.pk = t2.pk; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +SELECT t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 LEFT JOIN t2 ON t1.pk = t2.pk; +pk a b rnk +1 1 10 1 +2 1 10 2 +3 1 20 4 +4 2 20 5 +5 2 20 6 +6 2 30 7 +7 3 10 3 +8 3 30 8 +9 3 30 9 +SELECT SQL_BUFFER_RESULT t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 LEFT JOIN t2 ON t1.pk = t2.pk; +pk a b rnk +1 1 10 1 +2 1 10 2 +3 1 20 4 +4 2 20 5 +5 2 20 6 +6 2 30 7 +7 3 10 3 +8 3 30 8 +9 3 30 9 +DROP TABLE t2; +EXPLAIN EXTENDED SELECT d.pk, d.a, d.b, rank() OVER (PARTITION BY d.a ORDER BY d.b) AS rnk, dense_rank() OVER (PARTITION BY d.a ORDER BY d.b) AS drnk FROM (SELECT pk, a, b FROM t1 WHERE a > 1) AS d; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using where; Using filesort +SELECT d.pk, d.a, d.b, rank() OVER (PARTITION BY d.a ORDER BY d.b) AS rnk, dense_rank() OVER (PARTITION BY d.a ORDER BY d.b) AS drnk FROM (SELECT pk, a, b FROM t1 WHERE a > 1) AS d; +pk a b rnk drnk +4 2 20 1 1 +5 2 20 1 1 +6 2 30 3 2 +7 3 10 1 1 +8 3 30 2 2 +9 3 30 2 2 +SELECT SQL_BUFFER_RESULT d.pk, d.a, d.b, rank() OVER (PARTITION BY d.a ORDER BY d.b) AS rnk, dense_rank() OVER (PARTITION BY d.a ORDER BY d.b) AS drnk FROM (SELECT pk, a, b FROM t1 WHERE a > 1) AS d; +pk a b rnk drnk +4 2 20 1 1 +5 2 20 1 1 +6 2 30 3 2 +7 3 10 1 1 +8 3 30 2 2 +9 3 30 2 2 +EXPLAIN EXTENDED SELECT d.a, d.rnk FROM (SELECT a, rank() OVER (ORDER BY a) AS rnk FROM t1) AS d; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 9 100.00 +2 DERIVED t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +SELECT d.a, d.rnk FROM (SELECT a, rank() OVER (ORDER BY a) AS rnk FROM t1) AS d; +a rnk +1 1 +1 1 +1 1 +2 4 +2 4 +2 4 +3 7 +3 7 +3 7 +ANALYZE FORMAT=JSON SELECT pk, rank() OVER (ORDER BY pk) AS rnk FROM t1 LIMIT 2; +ANALYZE +{ + "query_optimization": { + "r_total_time_ms": "REPLACED" + }, + "query_block": { + "select_id": 1, + "cost": "REPLACED", + "r_loops": 1, + "r_total_time_ms": "REPLACED", + "nested_loop": [ + { + "table": { + "table_name": "t1", + "access_type": "index", + "key": "PRIMARY", + "key_length": "4", + "used_key_parts": ["pk"], + "loops": 1, + "r_loops": 1, + "rows": 9, + "r_rows": 2, + "cost": "REPLACED", + "r_table_time_ms": "REPLACED", + "r_other_time_ms": "REPLACED", + "r_engine_stats": REPLACED, + "filtered": 100, + "r_total_filtered": 100, + "r_filtered": 100, + "using_index": true + } + } + ] + } +} +CREATE TABLE tg (a INT, b INT, KEY(a, b)); +INSERT INTO tg VALUES (1, 1); +INSERT INTO tg VALUES (1, 2); +INSERT INTO tg VALUES (2, 1); +INSERT INTO tg VALUES (2, 2); +INSERT INTO tg VALUES (2, 3); +INSERT INTO tg VALUES (3, 1); +EXPLAIN EXTENDED SELECT a, b, rank() OVER (ORDER BY a) AS rnk, dense_rank() OVER (ORDER BY a) AS drnk FROM tg GROUP BY a, b; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE tg range NULL a 10 NULL 6 100.00 Using index for group-by +SELECT a, b, rank() OVER (ORDER BY a) AS rnk, dense_rank() OVER (ORDER BY a) AS drnk FROM tg GROUP BY a, b; +a b rnk drnk +1 1 1 1 +1 2 1 1 +2 1 3 2 +2 2 3 2 +2 3 3 2 +3 1 6 3 +SELECT SQL_BUFFER_RESULT a, b, rank() OVER (ORDER BY a) AS rnk, dense_rank() OVER (ORDER BY a) AS drnk FROM tg GROUP BY a, b; +a b rnk drnk +1 1 1 1 +1 2 1 1 +2 1 3 2 +2 2 3 2 +2 3 3 2 +3 1 6 3 +EXPLAIN EXTENDED SELECT a, rank() OVER (ORDER BY a, b) AS rnk, dense_rank() OVER (ORDER BY a, b) AS drnk FROM tg GROUP BY a; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE tg range NULL a 5 NULL 6 100.00 Using index for group-by +SELECT a, rank() OVER (ORDER BY a, b) AS rnk, dense_rank() OVER (ORDER BY a, b) AS drnk FROM tg GROUP BY a; +a rnk drnk +1 1 1 +2 2 2 +3 3 3 +SELECT SQL_BUFFER_RESULT a, rank() OVER (ORDER BY a, b) AS rnk, dense_rank() OVER (ORDER BY a, b) AS drnk FROM tg GROUP BY a; +a rnk drnk +1 1 1 +2 2 2 +3 3 3 +EXPLAIN EXTENDED SELECT a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM tg GROUP BY a, b WINDOW w AS (PARTITION BY a ORDER BY b); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE tg range NULL a 10 NULL 6 100.00 Using index for group-by +SELECT a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM tg GROUP BY a, b WINDOW w AS (PARTITION BY a ORDER BY b); +a b rn rnk drnk +1 1 1 1 1 +1 2 2 2 2 +2 1 1 1 1 +2 2 2 2 2 +2 3 3 3 3 +3 1 1 1 1 +SELECT SQL_BUFFER_RESULT a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM tg GROUP BY a, b WINDOW w AS (PARTITION BY a ORDER BY b); +a b rn rnk drnk +1 1 1 1 1 +1 2 2 2 2 +2 1 1 1 1 +2 2 2 2 2 +2 3 3 3 3 +3 1 1 1 1 +EXPLAIN EXTENDED SELECT rank() OVER (ORDER BY a), rank() OVER (ORDER BY b) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT rank() OVER (PARTITION BY max(a) ORDER BY b) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT max(a), rank() OVER (ORDER BY b) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT rank() OVER (ORDER BY a) FROM t1 GROUP BY a; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary; Using filesort +CREATE TABLE t2 (a INT, x INT, KEY(a)); +INSERT INTO t2 VALUES (1,10); +INSERT INTO t2 VALUES (1,20); +INSERT INTO t2 VALUES (2,20); +INSERT INTO t2 VALUES (2,30); +EXPLAIN SELECT tg.a, rank() OVER (ORDER BY tg.a) FROM tg JOIN t2 ON tg.a=t2.a GROUP BY tg.a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 index a a 5 NULL 4 Using where; Using index; Using temporary +1 SIMPLE tg ref a a 5 test.t2.a 1 Using index +EXPLAIN SELECT a, x FROM t2 FORCE INDEX FOR GROUP BY (a) GROUP BY a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 index NULL a 5 NULL 4 +EXPLAIN SELECT a, x, rank() OVER (ORDER BY a) FROM t2 FORCE INDEX FOR GROUP BY (a) GROUP BY a; +id select_type table type possible_keys key key_len ref rows Extra +1 SIMPLE t2 index NULL a 5 NULL 4 Using temporary +EXPLAIN EXTENDED SELECT rank() OVER (ORDER BY a) FROM tg GROUP BY 1+2; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE tg index NULL a 10 NULL 6 100.00 Using index; Using temporary +EXPLAIN EXTENDED SELECT pk, a, b FROM t1 GROUP BY pk; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using filesort +EXPLAIN EXTENDED SELECT pk, a, rank() OVER (ORDER BY pk) AS rnk FROM t1 GROUP BY pk; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary; Using filesort +EXPLAIN EXTENDED SELECT pk, rank() OVER (ORDER BY a) AS x FROM t1 ORDER BY x; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary; Using filesort +DROP TABLE tg, t2; +EXPLAIN EXTENDED SELECT count(*) OVER w, sum(b) OVER w, avg(b) OVER w, min(b) OVER w, max(b) OVER w FROM t1 WINDOW w AS (ORDER BY b, pk ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW); +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT sum(b) OVER (ORDER BY b) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT count(*) OVER (ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT count(*) OVER (ORDER BY a RANGE BETWEEN CURRENT ROW AND 5 FOLLOWING) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT sum(b) OVER () FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +EXPLAIN EXTENDED SELECT sum(b) OVER (PARTITION BY a) FROM t1; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 SIMPLE t1 ALL NULL NULL NULL NULL 9 100.00 Using temporary +DROP TABLE t1; diff --git a/mysql-test/main/win_streaming.test b/mysql-test/main/win_streaming.test new file mode 100644 index 0000000000000..b1aa065ead1ee --- /dev/null +++ b/mysql-test/main/win_streaming.test @@ -0,0 +1,266 @@ +# +# Streaming Window Functions Tests +# + +# I will remove these comments when I'm done testing, will just write the # cases I'll consider for testing here. + +# Explain per scenario to lock up streaming path +# Explains for non streamable cases, no need to check output + +# Order by and partition +# Explain with one function to show streamable and index or filesort is +# used +# Show compatible functions also stream +# show results only to prove partition tracking correctness +# windows reusing the main query order (whichever is longer) under its mdev +# incompatible orders materialize +# aggregate functions inside partition lists materialize +# aggregate functions anywhere in the select list materialize +# cases to look harder later (subqueries, expressions) + +# Test with limit and analyze to show we read only rows needed + +# For each streamable case we run the query twice: once as-is (streaming path) +# and once with SQL_BUFFER_RESULT, which forces a temp table and so the old +# materialized path. The two must agree, which is what proves the streamed +# values are correct. +# We wrap both in --sorted_result because the streaming path emits rows in the +# window's sort order while the buffered path emits them from the temp table, +# so the row order can differ even when every value matches. Sorting both and +# selecting the key columns (pk,a,b) lets us compare them as multisets. + +# I add this because EXPLAIN EXTENDED emits a Note 1003 with the reconstructed query for every +# statement, which I think is not necessary and clutters result. +--disable_warnings + +CREATE TABLE t1 (pk INT PRIMARY KEY, a INT, b INT); +INSERT INTO t1 VALUES (1, 1, 10); +INSERT INTO t1 VALUES (2, 1, 10); +INSERT INTO t1 VALUES (3, 1, 20); +INSERT INTO t1 VALUES (4, 2, 20); +INSERT INTO t1 VALUES (5, 2, 20); +INSERT INTO t1 VALUES (6, 2, 30); +INSERT INTO t1 VALUES (7, 3, 10); +INSERT INTO t1 VALUES (8, 3, 30); +INSERT INTO t1 VALUES (9, 3, 30); + +--let $q= pk, a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (PARTITION BY a ORDER BY b, pk) +eval EXPLAIN EXTENDED SELECT $q; +eval EXPLAIN EXTENDED SELECT SQL_BUFFER_RESULT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Tests peer handling as duplicates exist when pk is not in the order list. +# row_number() is dropped here because the number assigned to a given row can differ between streaming and materialization. +# Filesort is not stable and there is no tie breaking. +--let $q= pk, a, b, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a) +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Order-only (no partition): all three functions, total order. +--let $q= pk, a, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM t1 WINDOW w AS (ORDER BY a, pk) +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Windows reusing the main query order (whichever order is longer is used for a +# single sort). We use EXPLAIN FORMAT=JSON here to show which sort key is used. +# window order longer than main ORDER BY +--source include/explain-no-costs.inc +--let $q= pk, a, b, rank() OVER (ORDER BY a, b) AS rnk FROM t1 ORDER BY a +eval EXPLAIN FORMAT=JSON SELECT $q; +eval SELECT $q; +eval SELECT SQL_BUFFER_RESULT $q; + +# main ORDER BY longer than window order +--source include/explain-no-costs.inc +--let $q= pk, a, b, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a, b +eval EXPLAIN FORMAT=JSON SELECT $q; +eval SELECT $q; +eval SELECT SQL_BUFFER_RESULT $q; + +# window order equals main ORDER BY +--source include/explain-no-costs.inc +--let $q= pk, a, rank() OVER (ORDER BY a) AS rnk FROM t1 ORDER BY a +eval EXPLAIN FORMAT=JSON SELECT $q; +eval SELECT $q; +eval SELECT SQL_BUFFER_RESULT $q; + +# Window functions inside expressions still stream as long as they're not +# aggregate functions that already require materialization. +--let $q= pk, a, rank() OVER w AS r, rank() OVER w + 1 AS r_plus, rank() OVER w - dense_rank() OVER w AS diff FROM t1 WINDOW w AS (ORDER BY a) +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Multi-table joins +CREATE TABLE t2 (pk INT PRIMARY KEY, c INT); +INSERT INTO t2 VALUES (1, 100); +INSERT INTO t2 VALUES (2, 200); +INSERT INTO t2 VALUES (3, 300); +INSERT INTO t2 VALUES (4, 400); +INSERT INTO t2 VALUES (5, 500); +INSERT INTO t2 VALUES (6, 600); +INSERT INTO t2 VALUES (7, 700); +INSERT INTO t2 VALUES (8, 800); +INSERT INTO t2 VALUES (9, 900); + +--let $q= t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 JOIN t2 ON t1.pk = t2.pk +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +--let $q= t1.pk, t1.a, t1.b, rank() OVER (ORDER BY t1.b, t1.pk) AS rnk FROM t1 LEFT JOIN t2 ON t1.pk = t2.pk +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +DROP TABLE t2; + +# Derived table in FROM: window functions in the outer query over a subquery. +--let $q= d.pk, d.a, d.b, rank() OVER (PARTITION BY d.a ORDER BY d.b) AS rnk, dense_rank() OVER (PARTITION BY d.a ORDER BY d.b) AS drnk FROM (SELECT pk, a, b FROM t1 WHERE a > 1) AS d +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Window function INSIDE a subquery +EXPLAIN EXTENDED SELECT d.a, d.rnk FROM (SELECT a, rank() OVER (ORDER BY a) AS rnk FROM t1) AS d; +SELECT d.a, d.rnk FROM (SELECT a, rank() OVER (ORDER BY a) AS rnk FROM t1) AS d; + +# r_rows should be equal to the limit. +--source include/analyze-format.inc +ANALYZE FORMAT=JSON SELECT pk, rank() OVER (ORDER BY pk) AS rnk FROM t1 LIMIT 2; + +# GROUP BY can stream: when the rows come out of the join already +# grouped (in the case of a single table loose index scan), and +# if the window order is compatible with the group list the window +# functions are computed on the streamed grouped rows. +# Note that this applies even if the window function order list is longer than the group list. +# As long as the group list is a prefix of the longest window order list. +CREATE TABLE tg (a INT, b INT, KEY(a, b)); +INSERT INTO tg VALUES (1, 1); +INSERT INTO tg VALUES (1, 2); +INSERT INTO tg VALUES (2, 1); +INSERT INTO tg VALUES (2, 2); +INSERT INTO tg VALUES (2, 3); +INSERT INTO tg VALUES (3, 1); + +# GROUP BY longer than the window order +--let $q= a, b, rank() OVER (ORDER BY a) AS rnk, dense_rank() OVER (ORDER BY a) AS drnk FROM tg GROUP BY a, b +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Group by shorter than the window order +--let $q= a, rank() OVER (ORDER BY a, b) AS rnk, dense_rank() OVER (ORDER BY a, b) AS drnk FROM tg GROUP BY a +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# Loose index scan with partition +--let $q= a, b, row_number() OVER w AS rn, rank() OVER w AS rnk, dense_rank() OVER w AS drnk FROM tg GROUP BY a, b WINDOW w AS (PARTITION BY a ORDER BY b) +eval EXPLAIN EXTENDED SELECT $q; +--sorted_result +eval SELECT $q; +--sorted_result +eval SELECT SQL_BUFFER_RESULT $q; + +# +# Cases that fall back to materialization +# + +# Incompatible orders between the two functions +EXPLAIN EXTENDED SELECT rank() OVER (ORDER BY a), rank() OVER (ORDER BY b) FROM t1; + +# Aggregate inside the PARTITION BY list +EXPLAIN EXTENDED SELECT rank() OVER (PARTITION BY max(a) ORDER BY b) FROM t1; + +# A non-window aggregate anywhere in the select list +EXPLAIN EXTENDED SELECT max(a), rank() OVER (ORDER BY b) FROM t1; + +# GROUP BY with no usable index needs a temp table for the grouping, materialize +EXPLAIN EXTENDED SELECT rank() OVER (ORDER BY a) FROM t1 GROUP BY a; + +CREATE TABLE t2 (a INT, x INT, KEY(a)); +INSERT INTO t2 VALUES (1,10); +INSERT INTO t2 VALUES (1,20); +INSERT INTO t2 VALUES (2,20); +INSERT INTO t2 VALUES (2,30); + +# GROUP BY across a multi-table join +EXPLAIN SELECT tg.a, rank() OVER (ORDER BY tg.a) FROM tg JOIN t2 ON tg.a=t2.a GROUP BY tg.a; + +# GROUP BY that uses a tight index scan (select list is not satisfied by the index) +EXPLAIN SELECT a, x FROM t2 FORCE INDEX FOR GROUP BY (a) GROUP BY a; +EXPLAIN SELECT a, x, rank() OVER (ORDER BY a) FROM t2 FORCE INDEX FOR GROUP BY (a) GROUP BY a; + +# Implicit/constant grouping (GROUP BY a constant expression) collapses to a +# single group (grouping optimized away) and does not stream +EXPLAIN EXTENDED SELECT rank() OVER (ORDER BY a) FROM tg GROUP BY 1+2; + +# GROUP BY on a unique NOT NULL index (here the PRIMARY KEY) is optimized away: +# every group is exactly one row, so no grouping operation is performed and the +# rows go straight through end_send() with no temp table (the GROUP BY is just +# rewritten to an ORDER BY). Shown here without a window function: +EXPLAIN EXTENDED SELECT pk, a, b FROM t1 GROUP BY pk; +# Adding a window function disables that unique-index optimization for not +# so the GROUP BY is kept. (This is not yet fixed for streaming) +EXPLAIN EXTENDED SELECT pk, a, rank() OVER (ORDER BY pk) AS rnk FROM t1 GROUP BY pk; + +# Ordering the outer query by the window function value, it needs to save the value first. +EXPLAIN EXTENDED SELECT pk, rank() OVER (ORDER BY a) AS x FROM t1 ORDER BY x; + +DROP TABLE tg, t2; + +# +# Aggregate window functions: not streamable yet. +# +# Aggregates return is_streamable() == false today, so every case below falls +# back and shows "Using temporary". They split into two groups: the cumulative +# running frame is only blocked by that is_streamable() gate and is intended to +# stream once aggregates are enabled; the rest fall back for frame reasons that +# will remain even then. + +# Will stream once aggregates are enabled: an explicit ROWS ... CURRENT ROW +# frame makes "current row" a physical position, so row N depends only on rows +# <= N and an O(1) running accumulator is enough. The EXPLAIN EXTENDED will then +# show "Using filesort" and NOT "Using temporary". +EXPLAIN EXTENDED SELECT count(*) OVER w, sum(b) OVER w, avg(b) OVER w, min(b) OVER w, max(b) OVER w FROM t1 WINDOW w AS (ORDER BY b, pk ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW); + +# Default-frame: ORDER BY with no explicit frame defaults to RANGE, not ROWS. +# Under RANGE "current row" is the end of the peer group, so a row's value is +# unknown until the group is scanned ahead and buffered. Looks like the +# streamable ROWS form above but will not stream. +EXPLAIN EXTENDED SELECT sum(b) OVER (ORDER BY b) FROM t1; +# Explicit RANGE, same peer-group reason. +EXPLAIN EXTENDED SELECT count(*) OVER (ORDER BY a RANGE BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM t1; +# Bottom bound past CURRENT ROW needs lookahead regardless of units. +EXPLAIN EXTENDED SELECT count(*) OVER (ORDER BY a RANGE BETWEEN CURRENT ROW AND 5 FOLLOWING) FROM t1; + +# Those span whole partitions, the whole partition must be buffered before the row is emitted, +# hence no streaming. +EXPLAIN EXTENDED SELECT sum(b) OVER () FROM t1; +EXPLAIN EXTENDED SELECT sum(b) OVER (PARTITION BY a) FROM t1; + +DROP TABLE t1; + +--enable_warnings diff --git a/sql/item_sum.h b/sql/item_sum.h index 39ed79e7c0203..f1f1ac3863090 100644 --- a/sql/item_sum.h +++ b/sql/item_sum.h @@ -418,6 +418,7 @@ class Item_sum :public Item_func_or_sum Item_sum(THD *thd, Item_sum *item); enum Type type() const override { return SUM_FUNC_ITEM; } virtual enum Sumfunctype sum_func () const=0; + virtual inline bool is_streamable() const { return false; } bool is_aggr_sum_func() { switch (sum_func()) { diff --git a/sql/item_windowfunc.h b/sql/item_windowfunc.h index bff614372e2fc..994d85525a6c5 100644 --- a/sql/item_windowfunc.h +++ b/sql/item_windowfunc.h @@ -150,6 +150,8 @@ class Item_sum_row_number: public Item_sum_int return name; } + inline bool is_streamable() const override { return true; } + protected: Item *shallow_copy(THD *thd) const override { return get_item_copy(thd, this); } @@ -215,6 +217,8 @@ class Item_sum_rank: public Item_sum_int return name; } + inline bool is_streamable() const override { return true; } + void setup_window_func(THD *thd, Window_spec *window_spec) override; void cleanup() override @@ -290,6 +294,8 @@ class Item_sum_dense_rank: public Item_sum_int return name; } + inline bool is_streamable() const override { return true; } + void setup_window_func(THD *thd, Window_spec *window_spec) override; void cleanup() override diff --git a/sql/sql_lex.cc b/sql/sql_lex.cc index 86555657deea8..82e06d6005098 100644 --- a/sql/sql_lex.cc +++ b/sql/sql_lex.cc @@ -3793,7 +3793,7 @@ uint st_select_lex::get_cardinality_of_ref_ptrs_slice(uint order_group_num_arg) select_n_where_fields * winfunc_factor + order_group_num * 2 * winfunc_factor + hidden_bit_fields + - fields_in_window_functions + 1; + fields_in_window_functions + 1; // consider this case for streaming return n; } diff --git a/sql/sql_select.cc b/sql/sql_select.cc index b2c6929f20311..09b3bd2c94d45 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -26,6 +26,8 @@ */ #include "mariadb.h" +#include "my_dbug.h" +#include "sql_list.h" #include "sql_priv.h" #include "unireg.h" #include "sql_select.h" @@ -71,6 +73,9 @@ #include "opt_hints.h" #include "opt_group_by_cardinality.h" +#include "sql_window.h" +#include "item_windowfunc.h" + /* A key part number that means we're using a fulltext scan. @@ -228,6 +233,8 @@ static enum_nested_loop_state end_update(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); static enum_nested_loop_state end_unique_update(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); +static enum_nested_loop_state +end_compute_win_func(JOIN *join, JOIN_TAB *join_tab, bool end_of_records); static int join_read_const_table(THD *thd, JOIN_TAB *tab, POSITION *pos); static int join_read_system(JOIN_TAB *tab); @@ -1600,6 +1607,7 @@ JOIN::prepare(TABLE_LIST *tables_init, COND *conds_init, uint og_num, DBUG_RETURN(-1); thd->lex->current_select->context_analysis_place= save_place; + // this sets window functions up if (setup_without_group(thd, ref_ptrs, tables_list, select_lex->leaf_tables, fields_list, all_fields, &conds, order, group_list, @@ -1608,6 +1616,14 @@ JOIN::prepare(TABLE_LIST *tables_init, COND *conds_init, uint og_num, &hidden_group_fields)) DBUG_RETURN(-1); + // this needs to decide compatibility with main query sorting if exists + // but the actual setting of which is set before test_if_need_tmp_table() + if (select_lex->n_sum_items == select_lex->window_funcs.elements && + have_streaming_window_funcs(thd, select_lex->window_funcs, + win_func_longest_order, order, group_list, + streaming_wf_order_is_longer)) + streamable_window_funcs= true; + /* Permanently remove redundant parts from the query if 1) This is a subquery @@ -3333,14 +3349,17 @@ int JOIN::optimize_stage2() need_tmp= test_if_need_tmp_table(); /* - If window functions are present then we can't have simple_order set to - TRUE as the window function needs a temp table for computation. - ORDER BY is computed after the window function computation is done, so - the sort will be done on the temp table. + If window functions are present and not streamable, then we can't have + simple_order set to TRUE as the window function needs a temp table for + computation. In this case, ORDER BY is computed after the window function + computation is done, so the sort will be done on the temp table. */ - if (select_lex->have_window_funcs()) + if (select_lex->have_window_funcs() && !streamable_window_funcs) simple_order= FALSE; + if (!need_tmp && simple_order && streaming_wf_order_is_longer) + order= win_func_longest_order; + /* If the hint FORCE INDEX FOR ORDER BY/GROUP BY is used for the table whose columns are required to be returned in a sorted order, then @@ -3576,6 +3595,23 @@ int JOIN::optimize_stage2() if (make_aggr_tables_info()) DBUG_RETURN(1); + if (streamable_window_funcs && !need_tmp) + { + JOIN_TAB *last_real_tab= join_tab + exec_join_tab_cnt() - 1; + DBUG_ASSERT(last_real_tab->next_select == end_send); + + if (!(last_real_tab->window_funcs_streaming_step= + new Window_funcs_sort_streaming(thd))) + DBUG_RETURN(true); + if (last_real_tab->window_funcs_streaming_step->setup( + select_lex->window_funcs)) + DBUG_RETURN(true); + + last_real_tab->next_select= end_compute_win_func; + /* Count that we're using window functions. */ + status_var_increment(thd->status_var.feature_window_functions); + } + init_join_cache_and_keyread(); if (init_range_rowid_filters()) @@ -4327,7 +4363,7 @@ bool JOIN::make_aggr_tables_info() - duplicate value removal Both of these operations are done after window function computation step. */ - if (select_lex->window_funcs.elements) + if (select_lex->window_funcs.elements && need_tmp) { curr_tab= join_tab + total_join_tab_cnt(); if (!(curr_tab->window_funcs_step= new Window_funcs_computation)) @@ -16783,6 +16819,11 @@ void JOIN_TAB::cleanup() cache->free(); cache= 0; } + if (window_funcs_streaming_step) + { + window_funcs_streaming_step->cleanup(); + window_funcs_streaming_step= nullptr; + } limit= 0; // Free select that was created for filesort outside of create_sort_index if (filesort && filesort->select && !filesort->own_select) @@ -24732,7 +24773,6 @@ sub_select(JOIN *join,JOIN_TAB *join_tab,bool end_of_records) join_tab->loosescan_key_len); skip_over= TRUE; } - error= info->read_record(); if (skip_over && likely(!error)) @@ -24949,6 +24989,7 @@ evaluate_join_record(JOIN *join, JOIN_TAB *join_tab, { enum enum_nested_loop_state rc; /* A match from join_tab is found for the current partial join. */ + // this is the loop rc= (*join_tab->next_select)(join, join_tab+1, 0); join->thd->get_stmt_da()->inc_current_row_for_warning(); if (rc != NESTED_LOOP_OK && rc != NESTED_LOOP_NO_MORE_ROWS) @@ -26162,6 +26203,23 @@ end_send(JOIN *join, JOIN_TAB *join_tab, bool end_of_records) DBUG_RETURN(NESTED_LOOP_OK); } +enum_nested_loop_state end_compute_win_func(JOIN *join, JOIN_TAB *join_tab, + bool end_of_records) +{ + // this show call process_row with the current row, and the list of window + // functions, process row runs cursors for wfs on the current row (will + // partition trackers work?) + // Then end_send would call the window_func()->val_*() so we need phase + // computation to read the live value + // we don't even need to pass the row to the window function, because the + // add() functions read from the TABLE::record[0] directly, as we did + // NOT call split_sum_func(), so we still point to base table + DBUG_ENTER("end_compute_win_func"); + if (!end_of_records && + (join_tab - 1)->window_funcs_streaming_step->process_row()) + DBUG_RETURN(NESTED_LOOP_ERROR); + DBUG_RETURN(end_send(join, join_tab, end_of_records)); +} /* @brief diff --git a/sql/sql_select.h b/sql/sql_select.h index 7216938ebfd00..0a61f430065bc 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -200,6 +200,8 @@ enum join_type { JT_UNKNOWN,JT_SYSTEM,JT_CONST,JT_EQ_REF,JT_REF,JT_MAYBE_REF, class JOIN; +class Window_funcs_sort_streaming; + enum enum_nested_loop_state { NESTED_LOOP_KILLED= -2, NESTED_LOOP_ERROR= -1, @@ -533,6 +535,12 @@ typedef struct st_join_table { */ Window_funcs_computation* window_funcs_step; + /* + Non-NULL value means this join_tab (last real table) must do stream window + function computation before sending + */ + Window_funcs_sort_streaming *window_funcs_streaming_step; + /** List of topmost expressions in the select list. The *next* JOIN_TAB in the plan should use it to obtain correct values. Same applicable to @@ -1753,9 +1761,27 @@ class JOIN :public Sql_alloc */ Sql_cmd_dml *sql_cmd_dml; + /* + True if the query has window functions passing the streaming criteria, + defined by have_streaming_window_funcs() + Note: this does not guarantee they will be streamed, if the query requires + a temp table for any other reason, the window functions follow the + materialization path. + */ + bool streamable_window_funcs= false; + + /* + These are set in have_streaming_window_funcs(). + streaming_wf_order_is_longer is True if the partition + order list of the + longest window function is longer than AND compatible with the ORDER BY + clause of the main query. + */ + bool streaming_wf_order_is_longer= false; + ORDER *win_func_longest_order= NULL; + JOIN(THD *thd_arg, List &fields_arg, ulonglong select_options_arg, select_result *result_arg) - :fields_list(fields_arg) + : fields_list(fields_arg) { init(thd_arg, fields_arg, select_options_arg, result_arg); } @@ -1898,18 +1924,29 @@ class JOIN :public Sql_alloc - We are using an ORDER BY or GROUP BY on fields not in the first table - We are using different ORDER BY and GROUP BY orders - The user wants us to buffer the result. - - We are using WINDOW functions. - When the WITH ROLLUP modifier is present, we cannot skip temporary table - creation for the DISTINCT clause just because there are only const tables. + - We are using WINDOW functions that cannot be computed by streaming. + The streaming step attaches to end_send, so it is only viable when the + last next_select is end_send. We must fall back to a temp table when: + * the window functions fail the streaming criteria + (see have_streaming_window_funcs()), or + * there are no real tables to stream from (only_const_tables()), or + * the plan would run an executor-side grouping step (end_send_group) + rather than end_send: i.e. grouping was optimized away to a single + implicit group (group_optimized_away), or there is a GROUP BY not + satisfied by a loose index scan. */ bool test_if_need_tmp_table() { return ((const_tables != table_count && - ((select_distinct || !simple_order || !simple_group) || - (group_list && order) || - MY_TEST(select_options & OPTION_BUFFER_RESULT))) || + ((select_distinct || !simple_order || !simple_group) || + (group_list && order) || + MY_TEST(select_options & OPTION_BUFFER_RESULT))) || (rollup.state != ROLLUP::STATE_NONE && select_distinct) || - select_lex->have_window_funcs()); + (select_lex->have_window_funcs() && + (!streamable_window_funcs || only_const_tables() || + group_optimized_away || + (group_list && + !join_tab[const_tables].is_using_loose_index_scan())))); } bool choose_subquery_plan(table_map join_tables); void get_partial_cost_and_fanout(int end_tab_idx, diff --git a/sql/sql_window.cc b/sql/sql_window.cc index e5b4b4de7d644..037e4f76f705d 100644 --- a/sql/sql_window.cc +++ b/sql/sql_window.cc @@ -15,14 +15,19 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ #include "mariadb.h" +#include "mysql/plugin.h" #include "sql_parse.h" #include "sql_select.h" #include "sql_list.h" #include "item_windowfunc.h" #include "filesort.h" #include "sql_base.h" +#include "item.h" +#include #include "sql_window.h" +static ORDER *concat_order_lists(MEM_ROOT *mem_root, ORDER *list1, + ORDER *list2); bool Window_spec::check_window_names(List_iterator_fast &it) @@ -497,27 +502,37 @@ int compare_order_elements(ORDER *ord1, int weight1, return cmp > 0 ? CMP_GT : CMP_LT; } -static -int compare_order_lists(SQL_I_List *part_list1, - int spec_number1, - SQL_I_List *part_list2, - int spec_number2) +/* + Overloaded to take ORDER* objects instead of SQL_I_List* (the longest + wf order list, and the main query order list). + Note that we use -1 for the spec_number of the main query order list, as + window spec numbers start from 0. + Returns CMP_EQ if the lists are equal or NULL, CMP_LT_C if the first list is + NULL or a prefix of the second list, CMP_GT_C if the second list is NULL or + a prefix of the first list, and CMP_LT or CMP_GT otherwise. +*/ +static int compare_order_lists(ORDER *list1, int spec_number1, ORDER *list2, + int spec_number2) { - if (part_list1 == part_list2) + if (!list1 && !list2) return CMP_EQ; - ORDER *elem1= part_list1->first; - ORDER *elem2= part_list2->first; - for ( ; elem1 && elem2; elem1= elem1->next, elem2= elem2->next) + if (!list1) + return CMP_LT_C; + if (!list2) + return CMP_GT_C; + ORDER *elem1= list1; + ORDER *elem2= list2; + for (; elem1 && elem2; elem1= elem1->next, elem2= elem2->next) { int cmp; // remove all constants as we don't need them for comparision - while(elem1 && ((*elem1->item)->real_item())->const_item()) + while (elem1 && ((*elem1->item)->real_item())->const_item()) { elem1= elem1->next; continue; } - while(elem2 && ((*elem2->item)->real_item())->const_item()) + while (elem2 && ((*elem2->item)->real_item())->const_item()) { elem2= elem2->next; continue; @@ -526,8 +541,8 @@ int compare_order_lists(SQL_I_List *part_list1, if (!elem1 || !elem2) break; - if ((cmp= compare_order_elements(elem1, spec_number1, - elem2, spec_number2))) + if ((cmp= + compare_order_elements(elem1, spec_number1, elem2, spec_number2))) return cmp; } if (elem1) @@ -537,6 +552,14 @@ int compare_order_lists(SQL_I_List *part_list1, return CMP_EQ; } +static int compare_order_lists(SQL_I_List *part_list1, int spec_number1, + SQL_I_List *part_list2, int spec_number2) +{ + if (part_list1 == part_list2) + return CMP_EQ; + return compare_order_lists(part_list1->first, spec_number1, + part_list2->first, spec_number2); +} static int compare_window_frame_bounds(Window_frame_bound *win_frame_bound1, @@ -781,6 +804,152 @@ void order_window_funcs_by_window_specs(List *win_func_list) } } +/* + Returns true if the window frame is unbounded preceding or current row. +*/ +static inline bool frame_is_streaming_compatible(Window_spec *win_spec) +{ + Window_frame *frame= win_spec->window_frame; + if (!frame) + return true; + if (frame->units != Window_frame::Frame_units::UNITS_ROWS) + return false; + bool unbounded_preceding_or_current= + frame->top_bound->precedence_type == Window_frame_bound::CURRENT || + (frame->top_bound->precedence_type == Window_frame_bound::PRECEDING && + frame->top_bound->is_unbounded()); + return (unbounded_preceding_or_current && + win_spec->window_frame->bottom_bound->precedence_type == + Window_frame_bound::CURRENT); +} + +static inline bool check_argument_list_aggregation(Window_spec *win_spec) +{ + for (ORDER *o= win_spec->partition_list->first; o; o= o->next) + if ((*o->item)->with_sum_func()) + return true; + + for (ORDER *o= win_spec->order_list->first; o; o= o->next) + if ((*o->item)->with_sum_func()) + return true; + + return false; +} + +static Item_window_func * +find_longest_compatible_order(const List &win_funcs) +{ + if (win_funcs.elements == 0) + return nullptr; + int longest_order_elements= -1; + Item_window_func *longest, *win_func; + List tmp_win_funcs= win_funcs; + List_iterator_fast it(tmp_win_funcs); + while ((win_func= it++)) + { + Window_spec *spec= win_func->window_spec; + int win_func_order_elements= + spec->partition_list->elements + spec->order_list->elements; + if (win_func_order_elements > longest_order_elements) + { + longest_order_elements= win_func_order_elements; + longest= win_func; + } + } + it.rewind(); + + Window_spec *longest_spec= longest->window_spec; + longest_spec->join_partition_and_order_lists(); + + // Check compatibility with other window function frames + while ((win_func= it++)) + { + if (win_func == longest) + continue; + Window_spec *spec= win_func->window_spec; + spec->join_partition_and_order_lists(); + int cmp= compare_order_lists(longest_spec->partition_list, + longest_spec->win_spec_number, + spec->partition_list, spec->win_spec_number); + spec->disjoin_partition_and_order_lists(); + if (!(cmp == CMP_EQ || cmp == CMP_GT_C)) + { + longest= nullptr; + break; + } + } + longest_spec->disjoin_partition_and_order_lists(); + return longest; +} + +// 1. Checks if all window function orderings are compatible. +// 2. We check each fucntion from our subset or no (let it be rank and +// row_number for now) +// 3. frame only current row (normal), or unbounded preceding (for sum +// functions and stuff like that, can be skipped now) +// 4. Longest order is compatible with main query order (if exists) or not. +bool have_streaming_window_funcs(THD *thd, List &win_funcs, + ORDER *&longest_wf_order, + ORDER *main_query_order, + ORDER *main_query_group_list, + bool &streaming_wf_order_is_longer) +{ + if (win_funcs.elements == 0) + return false; + + Item_window_func *win_func_with_longest_order= + find_longest_compatible_order(win_funcs); + if (!win_func_with_longest_order) + return false; + + List_iterator_fast it(win_funcs); + Item_window_func *win_func; + int cmp; + + while ((win_func= it++)) + { + Window_spec *spec= win_func->window_spec; + if (check_argument_list_aggregation(spec) || + !(win_func->window_func()->is_streamable() && + frame_is_streaming_compatible(win_func->window_spec))) + return false; + } + + longest_wf_order= concat_order_lists( + thd->mem_root, + win_func_with_longest_order->window_spec->partition_list->first, + win_func_with_longest_order->window_spec->order_list->first); + + cmp= compare_order_lists( + longest_wf_order, + win_func_with_longest_order->window_spec->win_spec_number, + main_query_order, -1); + + if (!(CMP_LT_C <= cmp && cmp <= CMP_GT_C)) + return false; + if (cmp == CMP_GT_C) + streaming_wf_order_is_longer= true; + else + streaming_wf_order_is_longer= false; + + // Ordering keys after the complete GROUP BY key does not affect the ordering + // of the grouped result: there is exactly one row per group key, so a + // trailing key can never be reached as a tie-breaker. Hence it is safe to + // drop the trailing keys even if the window function references non-grouped + // columns, whose values are plan-dependent but cannot affect the ordering + // between grouped rows. (Assumes the whole GROUP BY key is matched as a + // prefix, and no WITH ROLLUP.) + if (main_query_group_list) + { + cmp= compare_order_lists( + longest_wf_order, + win_func_with_longest_order->window_spec->win_spec_number, + main_query_group_list, -1); + if (!(CMP_LT_C <= cmp && cmp <= CMP_GT_C)) + return false; + } + return true; +} ///////////////////////////////////////////////////////////////////////////// @@ -1258,8 +1427,6 @@ class Cursor_manager List cursors; }; - - ////////////////////////////////////////////////////////////////////////////// // RANGE-type frames ////////////////////////////////////////////////////////////////////////////// @@ -2908,7 +3075,8 @@ bool compute_window_func(THD *thd, tracker->init(); partition_trackers.push_back(tracker); } - + // the frame cursor thing i think would not need much change if we assume + // current frame = current row List_iterator_fast iter_part_trackers(partition_trackers); ha_rows rownum= 0; uchar *rowid_buf= (uchar*) my_malloc(PSI_INSTRUMENT_ME, tbl->file->ref_length, MYF(0)); @@ -2926,7 +3094,6 @@ bool compute_window_func(THD *thd, iter_win_funcs.rewind(); iter_part_trackers.rewind(); iter_cursor_managers.rewind(); - Group_bound_tracker *tracker; while ((win_func= iter_win_funcs++) && (tracker= iter_part_trackers++) && @@ -3065,6 +3232,7 @@ bool Window_func_runner::exec(THD *thd, TABLE *tbl, SORT_INFO *filesort_result) Item_window_func *win_func; while ((win_func= it++)) { + // i need this so it reads live not from result_field win_func->set_phase_to_computation(); // TODO(cvicentiu) Setting the aggregator should probably be done during // setup of Window_funcs_sort. @@ -3073,6 +3241,7 @@ bool Window_func_runner::exec(THD *thd, TABLE *tbl, SORT_INFO *filesort_result) } it.rewind(); + // i would skip this now List cursor_managers; if (get_window_functions_required_cursors(thd, window_functions, &cursor_managers)) @@ -3085,6 +3254,7 @@ bool Window_func_runner::exec(THD *thd, TABLE *tbl, SORT_INFO *filesort_result) tbl, filesort_result); while ((win_func= it++)) { + // we do not want this at all in streaming win_func->set_phase_to_retrieval(); } @@ -3123,7 +3293,8 @@ bool Window_funcs_sort::setup(THD *thd, SQL_SELECT *sel, JOIN_TAB *join_tab) { Window_spec *spec; - Item_window_func *win_func= it.peek(); + Item_window_func *win_func= it.peek(); + // reuse this Item_window_func *win_func_with_longest_order= NULL; int longest_order_elements= -1; @@ -3153,6 +3324,8 @@ bool Window_funcs_sort::setup(THD *thd, SQL_SELECT *sel, in a way that the result is valid for all window functions belonging to this Window_funcs_sort. */ + // all this i should have done earlier for streaming (on base table, or + // reusing the main query order (for later)) spec= win_func_with_longest_order->window_spec; ORDER* sort_order= concat_order_lists(thd->mem_root, @@ -3254,6 +3427,80 @@ void Window_funcs_computation::cleanup() } } +bool Window_funcs_sort_streaming::setup(List &window_funcs) +{ + order_window_funcs_by_window_specs(&window_funcs); + + List_iterator_fast it(window_funcs); + Item_window_func *win_func; + if (get_window_functions_required_cursors(thd, window_funcs, + &cursor_managers)) + return true; + + Group_bound_tracker *tracker; + while ((win_func= it++)) + { + tracker= + new Group_bound_tracker(thd, win_func->window_spec->partition_list); + tracker->init(); + partition_trackers.push_back(tracker); + + // So that end_send gets the live value of the window function on calling + // val_*(), and not the value from result_field. + win_func->set_phase_to_computation(); + + // sets peer tracker inside rank() + Item_sum *sum_func= win_func->window_func(); + sum_func->setup_window_func(thd, win_func->window_spec); + + // for handling aggregate functions (not done yet, still need to define + // frame for those). + win_func->window_func()->set_aggregator(thd, + Aggregator::SIMPLE_AGGREGATOR); + } + this->win_funcs= window_funcs; // internal variable points to the list + return false; +} + +bool Window_funcs_sort_streaming::process_row() +{ + List_iterator_fast iter_win_funcs(win_funcs); + List_iterator_fast iter_part_trackers( + partition_trackers); + List_iterator_fast iter_cursor_managers(cursor_managers); + Item_window_func *win_func; + Cursor_manager *cursor_manager; + Group_bound_tracker *tracker; + // i copied this for now from compute_window_func + while ((win_func= iter_win_funcs++) && (tracker= iter_part_trackers++) && + (cursor_manager= iter_cursor_managers++)) + { + if (tracker->check_if_next_group() || (rownum == 0)) + { + /* TODO(cvicentiu) + Clearing window functions should happen through cursors. */ + win_func->window_func()->clear(); + cursor_manager->notify_cursors_partition_changed(rownum); + } + else + { + cursor_manager->notify_cursors_next_row(); + } + + /* Check if we found any error in the window function while adding values + through cursors. */ + if (unlikely(thd->is_error() || thd->is_killed())) + return true; + } + rownum++; + return false; +} + +void Window_funcs_sort_streaming::cleanup() +{ + cursor_managers.delete_elements(); + partition_trackers.delete_elements(); +} Explain_aggr_window_funcs* Window_funcs_computation::save_explain_plan(MEM_ROOT *mem_root, diff --git a/sql/sql_window.h b/sql/sql_window.h index 7009b8895a667..f24daf24d9eea 100644 --- a/sql/sql_window.h +++ b/sql/sql_window.h @@ -20,7 +20,10 @@ #include "filesort.h" class Item_window_func; - +class Item_sum; +class Group_bound_tracker; +class Frame_cursor; +class Cursor_manager; /* Window functions module. @@ -181,6 +184,11 @@ int setup_windows(THD *thd, Ref_ptr_array ref_pointer_array, TABLE_LIST *tables, List &fields, List &all_fields, List &win_specs, List &win_funcs); +bool have_streaming_window_funcs(THD *thd, List &win_funcs, + ORDER *&longest_wf_order, + ORDER *main_query_order, + ORDER *main_query_group_list, + bool &streaming_wf_order_is_longer); ////////////////////////////////////////////////////////////////////////////// // Classes that make window functions computation a part of SELECT's query plan @@ -256,5 +264,27 @@ class Window_funcs_computation : public Sql_alloc void cleanup(); }; +class Window_funcs_sort_streaming : public Sql_alloc +{ +public: + Window_funcs_sort_streaming(THD *thd) : thd(thd) {} + bool setup(List &win_funcs); + /* + The object is attached to the last real JOIN_TAB in the query. This + function is called by end_compute_win_func() to run the window functions + computation over the current row in the JOIN output, assuming the row sits + in TABLE::record[0]. Then end_send calls val_*() methods of the window + functions to retrieve the live computed values and sends the row to output. + */ + bool process_row(); + void cleanup(); + +private: + int rownum= 0; // Internal state for process row + THD *thd= nullptr; + List win_funcs; + List cursor_managers; + List partition_trackers; +}; #endif /* SQL_WINDOW_INCLUDED */