From 5b1a6e7efc488e606d953f99699c49339b1d397b Mon Sep 17 00:00:00 2001 From: liu qiren <13559537330@163.com> Date: Thu, 30 Apr 2026 15:36:39 +0800 Subject: [PATCH] Feature: reject queries that cause partition fullscan Add a plan-time check that rejects queries on partitioned tables when no effective partition pruning occurs, preventing unintended full partition scans that waste cluster resources. New GUC parameters: - reject_partition_fullscan (bool, default on): enable/disable - partition_fullscan_threshold (int, default 0): max partitions allowed after pruning. 0 = reject only true fullscans. Planner path (inherit.c): - Check in expand_partitioned_rtentry() after prune_append_rel_partitions() returns - Exempts parameterized queries (Param nodes in baserestrictinfo) ORCA path (orca.c): - Post-plan check in optimize_query() using plan_tree_walker - Inspects part_prune_info on 7 Dynamic scan node types - Skips PartitionSelector (JOIN dynamic pruning) - Exempts nodes with initial/exec_pruning_steps Exemptions: enable_partition_pruning=off, single-partition tables, parameterized queries, JOIN-based dynamic partition selection. --- src/backend/optimizer/path/costsize.c | 2 + src/backend/optimizer/plan/orca.c | 164 ++++++++++++++++++ src/backend/optimizer/util/inherit.c | 85 +++++++++ src/backend/utils/misc/guc.c | 27 +++ src/include/optimizer/cost.h | 2 + src/include/utils/unsync_guc_name.h | 2 + src/test/regress/greenplum_schedule | 2 + .../regress/sql/partition_fullscan_reject.sql | 127 ++++++++++++++ 8 files changed, 411 insertions(+) create mode 100644 src/test/regress/sql/partition_fullscan_reject.sql diff --git a/src/backend/optimizer/path/costsize.c b/src/backend/optimizer/path/costsize.c index 4a1bbbf584b..9f275f9b54a 100644 --- a/src/backend/optimizer/path/costsize.c +++ b/src/backend/optimizer/path/costsize.c @@ -160,6 +160,8 @@ bool enable_parallel_append = true; bool enable_parallel_hash = true; bool enable_partition_pruning = true; bool enable_async_append = true; +bool reject_partition_fullscan = true; +int partition_fullscan_threshold = 0; typedef struct { diff --git a/src/backend/optimizer/plan/orca.c b/src/backend/optimizer/plan/orca.c index 514385cc2e9..bd8eca187ba 100644 --- a/src/backend/optimizer/plan/orca.c +++ b/src/backend/optimizer/plan/orca.c @@ -21,6 +21,7 @@ #include "postgres.h" +#include "cdb/cdbllize.h" #include "cdb/cdbmutate.h" /* apply_shareinput */ #include "cdb/cdbplan.h" #include "cdb/cdbvars.h" @@ -45,6 +46,8 @@ #include "utils/syscache.h" #include "catalog/pg_proc.h" #include "catalog/pg_namespace.h" +#include "optimizer/cost.h" +#include "optimizer/walkers.h" /* GPORCA entry point */ extern PlannedStmt * GPOPTOptimizedPlan(Query *parse, bool *had_unexpected_failure, OptimizerOptions *opts); @@ -53,6 +56,7 @@ static Plan *remove_redundant_results(PlannerInfo *root, Plan *plan); static Node *remove_redundant_results_mutator(Node *node, void *); static bool can_replace_tlist(Plan *plan); static Node *push_down_expr_mutator(Node *node, List *child_tlist); +static void check_partition_fullscan(PlannedStmt *stmt); /* * Logging of optimization outcome @@ -192,6 +196,163 @@ query_contains_support_functions(Query *query) return query_or_expression_tree_walker((Node *) query, check_support_functions_walker, &context, 0); } +/* + * get_part_prune_info + * Extract part_prune_info from a plan node, if present. + * Returns NULL for node types without partition pruning info. + * PartitionSelector is always skipped (JOIN-based dynamic pruning). + */ +static PartitionPruneInfo * +get_part_prune_info(Plan *plan) +{ + switch (nodeTag(plan)) + { + case T_Append: + return ((Append *) plan)->part_prune_info; + case T_MergeAppend: + return ((MergeAppend *) plan)->part_prune_info; + case T_DynamicSeqScan: + return ((DynamicSeqScan *) plan)->part_prune_info; + case T_DynamicIndexScan: + return ((DynamicIndexScan *) plan)->part_prune_info; + case T_DynamicIndexOnlyScan: + return ((DynamicIndexOnlyScan *) plan)->part_prune_info; + case T_DynamicBitmapHeapScan: + return ((DynamicBitmapHeapScan *) plan)->part_prune_info; + case T_DynamicForeignScan: + return ((DynamicForeignScan *) plan)->part_prune_info; + case T_PartitionSelector: + /* Skip: JOIN dynamic pruning, present_parts is always full */ + return NULL; + default: + return NULL; + } +} + +/* + * check_partition_fullscan_in_node + * Check a single plan node's partition pruning info against the + * rejection policy. Raises ERROR if the query would scan too many + * partitions without effective pruning. + */ +static void +check_partition_fullscan_in_node(PartitionPruneInfo *ppi, List *rtable) +{ + ListCell *lc1; + + if (ppi == NULL) + return; + + foreach(lc1, ppi->prune_infos) + { + List *prune_info_list = (List *) lfirst(lc1); + ListCell *lc2; + + foreach(lc2, prune_info_list) + { + PartitionedRelPruneInfo *pinfo = + (PartitionedRelPruneInfo *) lfirst(lc2); + int total = pinfo->nparts; + int present = bms_num_members(pinfo->present_parts); + int threshold = partition_fullscan_threshold; + bool do_reject = false; + + /* Skip single-partition tables */ + if (total <= 1) + continue; + + /* + * If initial_pruning_steps or exec_pruning_steps is not empty, + * the executor can still prune partitions at startup or runtime + * (e.g., parameterized queries). Do not reject. + */ + if (pinfo->initial_pruning_steps != NIL || + pinfo->exec_pruning_steps != NIL) + continue; + + if (threshold == 0) + do_reject = (present == total); + else + do_reject = (present > threshold); + + if (do_reject) + { + RangeTblEntry *rte = rt_fetch(pinfo->rtindex, rtable); + char *relname = get_rel_name(rte->relid); + char *nspname = get_namespace_name( + get_rel_namespace(rte->relid)); + + ereport(ERROR, + (errcode(ERRCODE_STATEMENT_TOO_COMPLEX), + errmsg("partitioned table \"%s.%s\" full partition " + "scan is not allowed, %d partitions would " + "be scanned", + nspname, relname, present), + errhint("Add a WHERE clause on the partition key " + "to enable partition pruning."))); + } + } + } +} + +/* + * Context for check_partition_fullscan_walker. + * Must begin with plan_tree_base_prefix as required by plan_tree_walker. + */ +typedef struct check_partition_fullscan_context +{ + plan_tree_base_prefix base; /* Required prefix */ + List *rtable; +} check_partition_fullscan_context; + +/* + * check_partition_fullscan_walker + * plan_tree_walker callback that visits each plan node and checks + * for partition fullscan violations. + */ +static bool +check_partition_fullscan_walker(Node *node, void *context) +{ + check_partition_fullscan_context *ctx = + (check_partition_fullscan_context *) context; + + if (node == NULL) + return false; + + if (is_plan_node(node)) + { + PartitionPruneInfo *ppi = get_part_prune_info((Plan *) node); + + check_partition_fullscan_in_node(ppi, ctx->rtable); + } + + return plan_tree_walker(node, check_partition_fullscan_walker, + context, true); +} + +/* + * check_partition_fullscan + * Entry point: check all partition scan nodes in a PlannedStmt + * for fullscan violations. + */ +static void +check_partition_fullscan(PlannedStmt *stmt) +{ + check_partition_fullscan_context ctx; + + if (!reject_partition_fullscan || !enable_partition_pruning) + return; + + if (stmt->planTree == NULL) + return; + + exec_init_plan_tree_base(&ctx.base, stmt); + ctx.rtable = stmt->rtable; + + /* Walk main plan tree (recurse_into_subplans handles SubPlan nodes) */ + check_partition_fullscan_walker((Node *) stmt->planTree, &ctx); +} + /* * optimize_query * Plan the query using the GPORCA planner @@ -293,6 +454,9 @@ optimize_query(Query *parse, int cursorOptions, ParamListInfo boundParams, Optim if (!result) return NULL; + /* Check for partition fullscan violations in ORCA-generated plan */ + check_partition_fullscan(result); + /* * Post-process the plan. */ diff --git a/src/backend/optimizer/util/inherit.c b/src/backend/optimizer/util/inherit.c index 1747cda95c7..8d6c5e8c89d 100644 --- a/src/backend/optimizer/util/inherit.c +++ b/src/backend/optimizer/util/inherit.c @@ -32,6 +32,9 @@ #include "parser/parsetree.h" #include "partitioning/partdesc.h" #include "partitioning/partprune.h" +#include "nodes/nodeFuncs.h" +#include "optimizer/cost.h" +#include "utils/lsyscache.h" #include "utils/rel.h" @@ -49,6 +52,47 @@ static Bitmapset *translate_col_privs(const Bitmapset *parent_privs, List *translated_vars); static void expand_appendrel_subquery(PlannerInfo *root, RelOptInfo *rel, RangeTblEntry *rte, Index rti); +static bool contain_param_walker(Node *node, void *context); +static bool restrictinfo_has_param(List *restrictinfos); + + +/* + * contain_param_walker + * Return true if expression tree contains any Param node. + * Covers PARAM_EXTERN (prepared statements) and PARAM_EXEC (subqueries). + */ +static bool +contain_param_walker(Node *node, void *context) +{ + if (node == NULL) + return false; + if (IsA(node, Param)) + return true; + return expression_tree_walker(node, contain_param_walker, context); +} + +/* + * restrictinfo_has_param + * Check if any RestrictInfo in the list contains Param nodes. + * + * Queries with parameterized conditions (e.g., prepared statements with $1) + * cannot prune partitions at plan time, but execution-time pruning may still + * reduce the partition set. We should not reject such queries. + */ +static bool +restrictinfo_has_param(List *restrictinfos) +{ + ListCell *lc; + + foreach(lc, restrictinfos) + { + RestrictInfo *rinfo = lfirst_node(RestrictInfo, lc); + + if (contain_param_walker((Node *) rinfo->clause, NULL)) + return true; + } + return false; +} /* @@ -352,6 +396,47 @@ expand_partitioned_rtentry(PlannerInfo *root, RelOptInfo *relinfo, /* Expand simple_rel_array and friends to hold child objects. */ num_live_parts = bms_num_members(live_parts); + + /* + * If reject_partition_fullscan is enabled, reject queries where + * partition pruning did not effectively reduce the partition set. + * + * Exemptions: + * - enable_partition_pruning is off (user explicitly disabled pruning) + * - Table has only 1 partition (nothing meaningful to prune) + * - Restriction clauses contain Param nodes (execution-time pruning + * may still reduce partitions at runtime) + */ + if (reject_partition_fullscan && + enable_partition_pruning && + relinfo->nparts > 1 && + !restrictinfo_has_param(relinfo->baserestrictinfo)) + { + int threshold = partition_fullscan_threshold; + bool do_reject = false; + + if (threshold == 0) + do_reject = (num_live_parts == relinfo->nparts); + else + do_reject = (num_live_parts > threshold); + + if (do_reject) + { + char *relname = get_rel_name(parentrte->relid); + char *nspname = get_namespace_name( + get_rel_namespace(parentrte->relid)); + + ereport(ERROR, + (errcode(ERRCODE_STATEMENT_TOO_COMPLEX), + errmsg("partitioned table \"%s.%s\" full partition " + "scan is not allowed, %d partitions would " + "be scanned", + nspname, relname, num_live_parts), + errhint("Add a WHERE clause on the partition key " + "to enable partition pruning."))); + } + } + if (num_live_parts > 0) expand_planner_arrays(root, num_live_parts); diff --git a/src/backend/utils/misc/guc.c b/src/backend/utils/misc/guc.c index 354302afd54..b5dc8302298 100644 --- a/src/backend/utils/misc/guc.c +++ b/src/backend/utils/misc/guc.c @@ -1211,6 +1211,19 @@ static struct config_bool ConfigureNamesBool[] = true, NULL, NULL, NULL }, + { + {"reject_partition_fullscan", PGC_USERSET, QUERY_TUNING_METHOD, + gettext_noop("Rejects queries that scan all partitions without pruning."), + gettext_noop("When enabled, queries on partitioned tables that " + "cannot prune any partition will be rejected with " + "an error, requiring a WHERE clause on the " + "partition key."), + GUC_EXPLAIN + }, + &reject_partition_fullscan, + true, + NULL, NULL, NULL + }, { {"enable_async_append", PGC_USERSET, QUERY_TUNING_METHOD, gettext_noop("Enables the planner's use of async append plans."), @@ -2307,6 +2320,20 @@ static struct config_int ConfigureNamesInt[] = 13, 1, INT_MAX, NULL, NULL, NULL }, + { + {"partition_fullscan_threshold", PGC_USERSET, QUERY_TUNING_METHOD, + gettext_noop("Maximum partitions allowed after pruning before " + "rejecting a query."), + gettext_noop("When reject_partition_fullscan is on, queries are " + "rejected if remaining partitions after pruning " + "exceed this threshold. 0 means reject only when " + "no pruning occurs at all."), + GUC_EXPLAIN + }, + &partition_fullscan_threshold, + 0, 0, INT_MAX, + NULL, NULL, NULL + }, { {"geqo_threshold", PGC_USERSET, DEFUNCT_OPTIONS, gettext_noop("Unused. Syntax check only for PostgreSQL compatibility."), diff --git a/src/include/optimizer/cost.h b/src/include/optimizer/cost.h index ff8b9c591f0..17268236f8e 100644 --- a/src/include/optimizer/cost.h +++ b/src/include/optimizer/cost.h @@ -89,6 +89,8 @@ extern PGDLLIMPORT bool enable_parallel_append; extern PGDLLIMPORT bool enable_parallel_hash; extern PGDLLIMPORT bool enable_partition_pruning; extern PGDLLIMPORT bool enable_async_append; +extern PGDLLIMPORT bool reject_partition_fullscan; +extern PGDLLIMPORT int partition_fullscan_threshold; extern PGDLLIMPORT int constraint_exclusion; extern bool gp_enable_hashjoin_size_heuristic; /*CDB*/ diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 85ecb3548e6..c504f7063e4 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -507,6 +507,7 @@ "parallel_query_use_streaming_hashagg", "parallel_setup_cost", "parallel_tuple_cost", + "partition_fullscan_threshold", "password_encryption", "plan_cache_mode", "planner_work_mem", @@ -531,6 +532,7 @@ "recovery_target_time", "recovery_target_timeline", "recovery_target_xid", + "reject_partition_fullscan", "remove_temp_files_after_crash", "repl_catchup_within_range", "resource_cleanup_gangs_on_wait", diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 604616791c8..76973294816 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -217,6 +217,8 @@ test: bfv_joins bfv_subquery bfv_planner bfv_legacy bfv_temp bfv_dml # test tpcds query 04 test: tpcds_q04 +test: partition_fullscan_reject + test: qp_olap_mdqa qp_misc gp_recursive_cte qp_dml_joins qp_skew qp_select partition_prune_opfamily gp_tsrf qp_join_union_all qp_join_universal qp_rowsecurity qp_query_params qp_full_join test: qp_misc_jiras qp_with_clause qp_executor qp_olap_windowerr qp_olap_window qp_derived_table qp_bitmapscan qp_dropped_cols diff --git a/src/test/regress/sql/partition_fullscan_reject.sql b/src/test/regress/sql/partition_fullscan_reject.sql new file mode 100644 index 00000000000..1f764e29e1f --- /dev/null +++ b/src/test/regress/sql/partition_fullscan_reject.sql @@ -0,0 +1,127 @@ +-- +-- Test partition fullscan rejection feature +-- +-- This tests the reject_partition_fullscan GUC which prevents queries +-- from scanning all partitions of a partitioned table without pruning. +-- + +-- Force Postgres planner first for deterministic behavior +SET optimizer = off; + +-- Create test partitioned table with 3 range partitions +CREATE TABLE pfr_test (id int, dt date, val text) + PARTITION BY RANGE (dt); +CREATE TABLE pfr_test_p1 PARTITION OF pfr_test + FOR VALUES FROM ('2025-01-01') TO ('2025-04-01'); +CREATE TABLE pfr_test_p2 PARTITION OF pfr_test + FOR VALUES FROM ('2025-04-01') TO ('2025-07-01'); +CREATE TABLE pfr_test_p3 PARTITION OF pfr_test + FOR VALUES FROM ('2025-07-01') TO ('2025-10-01'); + +-- Single-partition table for exemption test +CREATE TABLE pfr_single (id int, dt date) + PARTITION BY RANGE (dt); +CREATE TABLE pfr_single_p1 PARTITION OF pfr_single + FOR VALUES FROM ('2025-01-01') TO ('2025-12-31'); + +-- ============================== +-- Test 1: Basic rejection - no WHERE clause +-- ============================== +SET reject_partition_fullscan = on; +SET partition_fullscan_threshold = 0; + +SELECT * FROM pfr_test; +SELECT count(*) FROM pfr_test; + +-- ============================== +-- Test 2: Pruning passes - WHERE on partition key +-- ============================== +SELECT * FROM pfr_test WHERE dt = '2025-02-01'; +SELECT * FROM pfr_test + WHERE dt >= '2025-01-01' AND dt < '2025-04-01'; + +-- ============================== +-- Test 3: WHERE not on partition key - should reject +-- ============================== +SELECT * FROM pfr_test WHERE val = 'x'; +SELECT * FROM pfr_test WHERE id = 1; + +-- ============================== +-- Test 4: WHERE 1=1 - should reject (constant folded to NIL) +-- ============================== +SELECT * FROM pfr_test WHERE 1 = 1; +SELECT * FROM pfr_test WHERE true; + +-- ============================== +-- Test 5: GUC off - allow full scan +-- ============================== +SET reject_partition_fullscan = off; +SELECT * FROM pfr_test; +SET reject_partition_fullscan = on; + +-- ============================== +-- Test 6: enable_partition_pruning=off exemption +-- ============================== +SET enable_partition_pruning = off; +SELECT * FROM pfr_test; +SET enable_partition_pruning = on; + +-- ============================== +-- Test 7: Single-partition table exemption +-- ============================== +SELECT * FROM pfr_single; + +-- ============================== +-- Test 8: Threshold mode +-- ============================== +SET partition_fullscan_threshold = 2; + +-- Pruned to 2 partitions, within threshold, should pass +SELECT * FROM pfr_test + WHERE dt >= '2025-01-01' AND dt < '2025-07-01'; + +-- All 3 partitions exceed threshold of 2, should reject +SELECT * FROM pfr_test; + +SET partition_fullscan_threshold = 0; + +-- ============================== +-- Test 9: Prepared statement with parameter (Param exemption) +-- ============================== +PREPARE pfr_q AS SELECT * FROM pfr_test WHERE dt = $1; +EXECUTE pfr_q('2025-02-01'); +DEALLOCATE pfr_q; + +-- ============================== +-- Test 10: UPDATE/DELETE without WHERE - should reject +-- ============================== +UPDATE pfr_test SET val = 'y'; +DELETE FROM pfr_test; + +-- UPDATE/DELETE with partition key - should pass +UPDATE pfr_test SET val = 'y' WHERE dt = '2025-02-01'; +DELETE FROM pfr_test WHERE dt = '2025-02-01'; + +-- ============================== +-- Test 11: Subquery containing partitioned table +-- ============================== +SELECT * FROM (SELECT * FROM pfr_test) sub; + +-- ============================== +-- Test 12: ORCA path +-- ============================== +SET optimizer = on; + +SELECT * FROM pfr_test; +SELECT * FROM pfr_test WHERE dt = '2025-02-01'; + +-- ============================== +-- Cleanup +-- ============================== +SET optimizer = off; +DROP TABLE pfr_test; +DROP TABLE pfr_single; +RESET optimizer; +RESET reject_partition_fullscan; +RESET partition_fullscan_threshold; +RESET enable_partition_pruning;