From 3aefe0cf7720f8c872f40f9a5ae7da3a293ab537 Mon Sep 17 00:00:00 2001 From: zhangwenchao <656540940@qq.com> Date: Fri, 6 Jun 2025 14:14:30 +0800 Subject: [PATCH 001/244] Change directory table directory's file path from relfilenode to relid. At first, we use relfilenode to construct the directory table's directory path. However, relfilenode may be different between QD and QE, this will cause the directory table's file paths are different which will be confused in using. As a result, we will use relid that is same between QD and QE to construct the directory table's file path in this commit. Authored-by: Zhang Wenchao zhangwenchao@apache.org --- src/backend/commands/dirtablecmds.c | 11 +---------- src/backend/storage/file/ufile.c | 12 ++++++------ src/include/storage/ufile.h | 4 ++-- 3 files changed, 9 insertions(+), 18 deletions(-) diff --git a/src/backend/commands/dirtablecmds.c b/src/backend/commands/dirtablecmds.c index 6f3bd94285e..7f34d46943d 100644 --- a/src/backend/commands/dirtablecmds.c +++ b/src/backend/commands/dirtablecmds.c @@ -136,21 +136,12 @@ CreateDirectoryTable(CreateDirectoryTableStmt *stmt, Oid relId) } else { - Form_pg_class pg_class_tuple = NULL; - HeapTuple class_tuple = NULL; RelFileNode relFileNode = {0}; - class_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relId)); - if (!HeapTupleIsValid(class_tuple)) - elog(ERROR, "cache lookup failed for relation %u", relId); - pg_class_tuple = (Form_pg_class) GETSTRUCT(class_tuple); - relFileNode.spcNode = spcId; relFileNode.dbNode = MyDatabaseId; - relFileNode.relNode = pg_class_tuple->relfilenode; - dirTablePath = UFileFormatPathName(&relFileNode); - ReleaseSysCache(class_tuple); + dirTablePath = UFileFormatPathName(relId, &relFileNode); } /* diff --git a/src/backend/storage/file/ufile.c b/src/backend/storage/file/ufile.c index 0c1ba28b995..0938a1bd83c 100644 --- a/src/backend/storage/file/ufile.c +++ b/src/backend/storage/file/ufile.c @@ -59,7 +59,7 @@ static int localFileRead(UFile *file, char *buffer, int amount); static int localFileWrite(UFile *file, char *buffer, int amount); static off_t localFileSize(UFile *file); static void localFileUnlink(Oid spcId, const char *fileName); -static char *localFormatPathName(RelFileNode *relFileNode); +static char *localFormatPathName(Oid relid, RelFileNode *relFileNode); static bool localEnsurePath(Oid spcId, const char *PathName); static bool localFileExists(Oid spcId, const char *fileName); static const char *localFileName(UFile *file); @@ -307,15 +307,15 @@ localFileUnlink(Oid spcId, const char *fileName) } static char * -localFormatPathName(RelFileNode *relFileNode) +localFormatPathName(Oid relid, RelFileNode *relFileNode) { if (relFileNode->spcNode == DEFAULTTABLESPACE_OID) return psprintf("base/%u/%u_dirtable", - relFileNode->dbNode, relFileNode->relNode); + relFileNode->dbNode, relid); else return psprintf("pg_tblspc/%u/%s/%u/%u_dirtable", relFileNode->spcNode, GP_TABLESPACE_VERSION_DIRECTORY, - relFileNode->dbNode, relFileNode->relNode); + relFileNode->dbNode, relid); } bool @@ -440,13 +440,13 @@ UFileUnlink(Oid spcId, const char *fileName) } char * -UFileFormatPathName(RelFileNode *relFileNode) +UFileFormatPathName(Oid relid, RelFileNode *relFileNode) { FileAm *fileAm; fileAm = GetTablespaceFileHandler(relFileNode->spcNode); - return fileAm->formatPathName(relFileNode); + return fileAm->formatPathName(relid, relFileNode); } bool diff --git a/src/include/storage/ufile.h b/src/include/storage/ufile.h index e34d66350ca..7e66196c040 100644 --- a/src/include/storage/ufile.h +++ b/src/include/storage/ufile.h @@ -44,7 +44,7 @@ typedef struct FileAm int (*write) (struct UFile *file, char *buffer, int amount); int64_t (*size) (struct UFile *file); void (*unlink) (Oid spcId, const char *fileName); - char* (*formatPathName) (RelFileNode *relFileNode); + char* (*formatPathName) (Oid relid, RelFileNode *relFileNode); bool (*ensurePath) (Oid spcId, const char *pathName); bool (*exists) (Oid spcId, const char *fileName); const char *(*name) (struct UFile *file); @@ -72,7 +72,7 @@ extern off_t UFileSize(UFile *file); extern const char *UFileName(UFile *file); extern void UFileUnlink(Oid spcId, const char *fileName); -extern char* UFileFormatPathName(RelFileNode *relFileNode); +extern char* UFileFormatPathName(Oid relid, RelFileNode *relFileNode); extern bool UFileEnsurePath(Oid spcId, const char *pathName); extern bool UFileExists(Oid spcId, const char *fileName); From f6ef0652735314d6e7b0254f2660040ffb5ec7d2 Mon Sep 17 00:00:00 2001 From: zhoujiaqi Date: Mon, 16 Jun 2025 09:51:57 +0800 Subject: [PATCH 002/244] PAX: disable build googlebench by default Change the `BUILD_GBENCH` to OFF by default, we should not build the google bench in release mode. --- contrib/pax_storage/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/pax_storage/CMakeLists.txt b/contrib/pax_storage/CMakeLists.txt index f4c132c1bd7..4d0cd0a495b 100644 --- a/contrib/pax_storage/CMakeLists.txt +++ b/contrib/pax_storage/CMakeLists.txt @@ -29,7 +29,7 @@ option(USE_PAX_CATALOG "Use manifest API, by pax impl" ON) # Build gtest options option(BUILD_GTEST "Build with google test" ON) -option(BUILD_GBENCH "Build with google benchmark" ON) +option(BUILD_GBENCH "Build with google benchmark" OFF) # Build pax tools option(BUILD_TOOLS "Build with pax tools" OFF) From 93fb6c5b825548e2f09b067beba5404babce9a1a Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Mon, 16 Jun 2025 19:39:28 -0700 Subject: [PATCH 003/244] Suppress deprecation warnings for MD5 functions on OpenSSL >= 3.0 (#1164) --- gpcontrib/gpcloud/src/s3utils.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/gpcontrib/gpcloud/src/s3utils.cpp b/gpcontrib/gpcloud/src/s3utils.cpp index bf1ffaba4e0..7445e4dbb16 100644 --- a/gpcontrib/gpcloud/src/s3utils.cpp +++ b/gpcontrib/gpcloud/src/s3utils.cpp @@ -103,6 +103,15 @@ size_t find_Nth(const string &str, // where to work return pos; } +#// Suppress deprecation warnings for MD5 functions on OpenSSL >= 3.0 +#if defined(__clang__) +#pragma clang diagnostic push +#pragma clang diagnostic ignored "-Wdeprecated-declarations" +#elif defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +#endif + MD5Calc::MD5Calc() { memset(this->md5, 0, MD5_DIGEST_STRING_LENGTH); MD5_Init(&this->c); @@ -126,6 +135,12 @@ const char *MD5Calc::Get() { return this->result.c_str(); } +#if defined(__clang__) +#pragma clang diagnostic pop +#elif defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + Config::Config(const string &filename, const string &url, const char *datadir) : _conf(NULL) { if (!url.empty()) { this->_conf = ini_load_from_url(url.c_str(), datadir); From 4e12400d0e1e646c3577b409ff41dd3e872139c5 Mon Sep 17 00:00:00 2001 From: zhoujiaqi Date: Thu, 8 May 2025 18:16:00 +0800 Subject: [PATCH 004/244] ORCA: Pruning the unused ColRefs in CTE For ORCA, it will prune the column reference(CColRef) which marked as "UNUSED". The producer and consumer output column reference of CTE will be marked as used, if current CTE have not been inlined. We observed that some SQL statements from our customers used CTE, but some of output columns were missing not used outside the consumer. Therefore, the current PR prunes the output columns that are not used by both the producer and the consumer. --- .../expected/shared_scan_optimizer.out | 7 +- .../gpopt/config/CConfigParamMapping.cpp | 4 + .../gpopt/translate/CTranslatorQueryToDXL.cpp | 6 +- .../libgpopt/include/gpopt/base/CCTEInfo.h | 27 +- .../libgpopt/include/gpopt/base/CColRef.h | 7 + .../libgpopt/include/gpopt/base/CUtils.h | 4 +- .../gpopt/operators/CPhysicalCTEProducer.h | 11 + .../gpopt/translate/CTranslatorDXLToExpr.h | 15 +- .../translate/CTranslatorDXLToExprUtils.h | 4 - .../gporca/libgpopt/src/base/CCTEInfo.cpp | 25 +- .../gporca/libgpopt/src/base/CColRef.cpp | 16 +- .../src/base/COptimizationContext.cpp | 2 +- .../libgpopt/src/base/CReqdPropPlan.cpp | 12 +- .../gporca/libgpopt/src/base/CUtils.cpp | 54 +- .../src/operators/CLogicalCTEConsumer.cpp | 45 + .../src/operators/CPhysicalCTEConsumer.cpp | 12 +- .../src/operators/CPhysicalCTEProducer.cpp | 33 +- .../src/operators/CPhysicalTableScan.cpp | 5 +- .../src/translate/CTranslatorDXLToExpr.cpp | 198 ++- .../translate/CTranslatorDXLToExprUtils.cpp | 31 - .../dxl/operators/CDXLLogicalCTEProducer.h | 10 +- .../naucrates/statistics/IStatistics.h | 6 + .../include/naucrates/traceflags/traceflags.h | 8 +- .../src/operators/CDXLLogicalCTEProducer.cpp | 4 +- .../CParseHandlerLogicalCTEProducer.cpp | 2 +- src/backend/utils/misc/guc_gp.c | 12 + src/include/utils/guc.h | 1 + src/include/utils/unsync_guc_name.h | 1 + src/test/regress/expected/cte_prune.out | 1198 +++++++++++++++ .../regress/expected/cte_prune_optimizer.out | 1295 +++++++++++++++++ .../expected/shared_scan_optimizer.out | 7 +- src/test/regress/sql/cte_prune.sql | 263 ++++ 32 files changed, 3214 insertions(+), 111 deletions(-) create mode 100644 src/test/regress/expected/cte_prune.out create mode 100644 src/test/regress/expected/cte_prune_optimizer.out create mode 100644 src/test/regress/sql/cte_prune.sql diff --git a/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out b/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out index a8ab8bd0c23..d782988bbd6 100644 --- a/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out @@ -107,12 +107,11 @@ WITH cte AS (SELECT * FROM t1 WHERE random() < 0.1 LIMIT 10) SELECT a, 1, 1 FROM Hash Cond: (share0_ref2.a = t2.a) -> Redistribute Motion 1:3 (slice3) Hash Key: share0_ref2.a - -> Result - -> Shared Scan (share slice:id 3:0) + -> Shared Scan (share slice:id 3:0) -> Hash -> Seq Scan on t2 - Optimizer: Pivotal Optimizer (GPORCA) -(17 rows) + Optimizer: GPORCA +(16 rows) -- This functions returns one more column than expected. CREATE OR REPLACE FUNCTION col_mismatch_func1() RETURNS TABLE (field1 int, field2 int) diff --git a/src/backend/gpopt/config/CConfigParamMapping.cpp b/src/backend/gpopt/config/CConfigParamMapping.cpp index b69d01e4074..2ca64bb959f 100644 --- a/src/backend/gpopt/config/CConfigParamMapping.cpp +++ b/src/backend/gpopt/config/CConfigParamMapping.cpp @@ -81,6 +81,10 @@ CConfigParamMapping::SConfigMappingElem CConfigParamMapping::m_elements[] = { false, // m_negate_param GPOS_WSZ_LIT("Prints optimization stats.")}, + {EopttraceDebugCTE, &optimizer_debug_cte, + false, // m_negate_param + GPOS_WSZ_LIT("Print debug info of CTE.")}, + {EopttraceMinidump, // GPDB_91_MERGE_FIXME: I turned optimizer_minidump from bool into // an enum-type GUC. It's a bit dirty to cast it like this.. diff --git a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp index 392f650664e..808277f5e0e 100644 --- a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp @@ -2577,7 +2577,8 @@ CTranslatorQueryToDXL::CreateDXLUnionAllForGroupingSets( GPOS_ASSERT(nullptr != m_dxl_cte_producers); CDXLLogicalCTEProducer *cte_prod_dxlop = GPOS_NEW(m_mp) - CDXLLogicalCTEProducer(m_mp, cte_id, op_colid_array_cte_producer); + CDXLLogicalCTEProducer(m_mp, cte_id, op_colid_array_cte_producer, + false /*could_be_pruned*/); CDXLNode *cte_producer_dxlnode = GPOS_NEW(m_mp) CDXLNode(m_mp, cte_prod_dxlop, select_project_join_dxlnode); m_dxl_cte_producers->Append(cte_producer_dxlnode); @@ -4785,7 +4786,8 @@ CTranslatorQueryToDXL::ConstructCTEProducerList(List *cte_list, CDXLLogicalCTEProducer *lg_cte_prod_dxlop = GPOS_NEW(m_mp) CDXLLogicalCTEProducer( - m_mp, m_context->m_cte_id_counter->next_id(), colid_array); + m_mp, m_context->m_cte_id_counter->next_id(), colid_array, + true /*could_be_pruned*/); CDXLNode *cte_producer_dxlnode = GPOS_NEW(m_mp) CDXLNode(m_mp, lg_cte_prod_dxlop, cte_child_dxlnode); diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h b/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h index 117769ddecb..831ee221373 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h @@ -219,6 +219,12 @@ class CCTEInfo : public CRefCount // consumers inside each cte/main query UlongToProducerConsumerMap *m_phmulprodconsmap; + // mappings CTE consumer ColId(Not DXL id) -> CTE producer Colref + UlongToColRefMap *m_phmcidcrCTE; + + // mappings CTE producer ColId(Not DXL id) -> CTE consumer Colref Array + UlongToColRefArrayMap *m_phmpidcrsCTE; + // initialize default statistics for a given CTE Producer void InitDefaultStats(CExpression *pexprCTEProducer); @@ -267,6 +273,24 @@ class CCTEInfo : public CRefCount // replace cte producer with given expression void ReplaceCTEProducer(CExpression *pexprCTEProducer); + // get the consumer colid -> producer colref map + UlongToColRefMap * + GetCTEConsumerMapping() const { + return m_phmcidcrCTE; + } + + // get the producer colid -> consumer colref array map + UlongToColRefArrayMap * + GetCTEProducerMapping() const { + return m_phmpidcrsCTE; + } + + // exist the CTE in global CTEInfo + BOOL + ExistCTE() const { + return m_ulNextCTEId > 0; + } + // next available CTE id ULONG next_id() @@ -316,7 +340,8 @@ class CCTEInfo : public CRefCount // return a map from Id's of consumer columns in the given column set to their corresponding producer columns UlongToColRefMap *PhmulcrConsumerToProducer(CMemoryPool *mp, ULONG ulCTEId, CColRefSet *pcrs, - CColRefArray *pdrgpcrProducer); + CColRefArray *pdrgpcrProducer, + ULongPtrArray *pdrgpcrUnusedProducer); }; // CCTEInfo } // namespace gpopt diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CColRef.h b/src/backend/gporca/libgpopt/include/gpopt/base/CColRef.h index e1da4d81769..af201349334 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CColRef.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CColRef.h @@ -36,6 +36,10 @@ using CColRef2dArray = CDynamicPtrArray; using UlongToColRefMap = CHashMap, gpos::Equals, CleanupDelete, CleanupNULL>; +// hash map mapping ULONG -> CColRefArray +using UlongToColRefArrayMap = + CHashMap, gpos::Equals, + CleanupDelete, CleanupRelease>; // hash map mapping ULONG -> const CColRef using UlongToConstColRefMap = CHashMap, gpos::Equals, @@ -82,6 +86,9 @@ class CColRef : public gpos::DbgPrintMixin // table info IMDId *m_mdid_table; + // debug function + const char* UsedStatusToString(EUsedStatus status) const; + public: CColRef(const CColRef &) = delete; diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CUtils.h b/src/backend/gporca/libgpopt/include/gpopt/base/CUtils.h index b1b55e29ea8..1e873e14847 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CUtils.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CUtils.h @@ -851,7 +851,9 @@ class CUtils // if the passed map is not null, mappings from old to copied variables are added to it static CColRefArray *PdrgpcrCopy( CMemoryPool *mp, CColRefArray *colref_array, BOOL fAllComputed = false, - UlongToColRefMap *colref_mapping = nullptr); + UlongToColRefMap *colref_mapping = nullptr, UlongToColRefMap *consumer_mapping = nullptr, + UlongToColRefArrayMap *producer_mapping = nullptr, + BOOL check_usage = false); // equality check between two arrays of column refs. Inputs can be NULL static BOOL Equals(CColRefArray *pdrgpcrFst, CColRefArray *pdrgpcrSnd); diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h index fec81912bb3..d9ed18b871c 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h @@ -37,6 +37,9 @@ class CPhysicalCTEProducer : public CPhysical // set representation of cte columns CColRefSet *m_pcrs; + // used to record unused columns in m_pdrgpcr + ULongPtrArray *m_pdrgpcr_unused; + public: CPhysicalCTEProducer(const CPhysicalCTEProducer &) = delete; @@ -73,6 +76,14 @@ class CPhysicalCTEProducer : public CPhysical return m_pdrgpcr; } + // used to make the CTE consumer colid get the right position in m_pdrgpcr. + // more details see the function CCTEInfo::PhmulcrConsumerToProducer() + ULongPtrArray * + PdrgpcrUnused() const + { + return m_pdrgpcr_unused; + } + // operator specific hash function ULONG HashValue() const override; diff --git a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h index b68f75ae874..47147f5bd39 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h +++ b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h @@ -59,7 +59,6 @@ using UlongToExprArrayMapIter = CHashMapIter, gpos::Equals, CleanupDelete, CleanupNULL>; - //--------------------------------------------------------------------------- // @class: // CTranslatorDXLToExpr @@ -364,6 +363,10 @@ class CTranslatorDXLToExpr // create an array of column references from an array of dxl column references CColRefArray *Pdrgpcr(const CDXLColDescrArray *dxl_col_descr_array); + // construct a dynamic array of column references corresponding to the + // given col ids + CColRefArray *Pdrgpcr(const ULongPtrArray *colids); + // construct the mapping between the DXL ColId and CColRef void ConstructDXLColId2ColRefMapping( const CDXLColDescrArray *dxl_col_descr_array, @@ -373,7 +376,15 @@ class CTranslatorDXLToExpr // look up the column reference in the hash map. We raise an exception if // the column is not found - static CColRef *LookupColRef(UlongToColRefMap *colref_mapping, ULONG colid); + CColRef *LookupColRef(ULONG colid, BOOL mark_used = true); + + // after the colref in CTE consumer marked used, the producer should marked + // the relatived colref as used. + void MarkCTEConsumerColAsUsed(UlongToColRefMap *cidcrCTE, UlongToColRefArrayMap *pidcrsCTE, ULONG colid); + + // after the colref in CTE producer marked used, the consumer should marked + // the relatived colref as used. + void MarkCTEProducerColAsUsed(UlongToColRefMap *cidcrCTE, UlongToColRefArrayMap *pidcrsCTE, ULONG colid); public: // ctor diff --git a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExprUtils.h b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExprUtils.h index e732f211e2e..73b5e054bb6 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExprUtils.h +++ b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExprUtils.h @@ -75,10 +75,6 @@ class CTranslatorDXLToExprUtils static CScalarBoolOp::EBoolOperator EBoolOperator( EdxlBoolExprType edxlbooltype); - // construct a dynamic array of col refs corresponding to the given col ids - static CColRefArray *Pdrgpcr(CMemoryPool *mp, - UlongToColRefMap *colref_mapping, - const ULongPtrArray *colids); // is the given expression is a scalar function that casts static BOOL FCastFunc(CMDAccessor *md_accessor, const CDXLNode *dxlnode, diff --git a/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp b/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp index be82b7882d6..335da5049fa 100644 --- a/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp +++ b/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp @@ -164,11 +164,16 @@ CCTEInfo::CCTEInfo(CMemoryPool *mp) : m_mp(mp), m_phmulcteinfoentry(nullptr), m_ulNextCTEId(0), - m_fEnableInlining(true) + m_fEnableInlining(true), + m_phmulprodconsmap(nullptr), + m_phmcidcrCTE(nullptr), + m_phmpidcrsCTE(nullptr) { GPOS_ASSERT(nullptr != mp); m_phmulcteinfoentry = GPOS_NEW(m_mp) UlongToCTEInfoEntryMap(m_mp); m_phmulprodconsmap = GPOS_NEW(m_mp) UlongToProducerConsumerMap(m_mp); + m_phmcidcrCTE = GPOS_NEW(m_mp) UlongToColRefMap(m_mp); + m_phmpidcrsCTE = GPOS_NEW(m_mp) UlongToColRefArrayMap(m_mp); } //--------------------------------------------------------------------------- @@ -183,6 +188,8 @@ CCTEInfo::~CCTEInfo() { CRefCount::SafeRelease(m_phmulcteinfoentry); CRefCount::SafeRelease(m_phmulprodconsmap); + CRefCount::SafeRelease(m_phmcidcrCTE); + CRefCount::SafeRelease(m_phmpidcrsCTE); } @@ -714,8 +721,9 @@ CCTEInfo::MarkUnusedCTEs() UlongToColRefMap * CCTEInfo::PhmulcrConsumerToProducer( CMemoryPool *mp, ULONG ulCTEId, - CColRefSet *pcrs, // set of columns to check - CColRefArray *pdrgpcrProducer // producer columns + CColRefSet *pcrs, // set of columns to check + CColRefArray *pdrgpcrProducer, // producer columns + ULongPtrArray *pdrgpcrUnusedProducer // producer unused columns ) { GPOS_ASSERT(nullptr != pcrs); @@ -727,13 +735,20 @@ CCTEInfo::PhmulcrConsumerToProducer( while (crsi.Advance()) { CColRef *colref = crsi.Pcr(); + // pruned colref no need do the mapping + if (colref->GetUsage() != CColRef::EUsed) { + continue; + } ULONG ulPos = UlConsumerColPos(ulCTEId, colref); if (gpos::ulong_max != ulPos) { - GPOS_ASSERT(ulPos < pdrgpcrProducer->Size()); + GPOS_ASSERT(ulPos < pdrgpcrUnusedProducer->Size()); + ULONG remapUlPos = ulPos - *(*pdrgpcrUnusedProducer)[ulPos]; + GPOS_ASSERT(remapUlPos < pdrgpcrProducer->Size()); - CColRef *pcrProducer = (*pdrgpcrProducer)[ulPos]; + CColRef *pcrProducer = (*pdrgpcrProducer)[remapUlPos]; + pcrProducer->MarkAsUsed(); BOOL fSuccess GPOS_ASSERTS_ONLY = colref_mapping->Insert( GPOS_NEW(mp) ULONG(colref->Id()), pcrProducer); GPOS_ASSERT(fSuccess); diff --git a/src/backend/gporca/libgpopt/src/base/CColRef.cpp b/src/backend/gporca/libgpopt/src/base/CColRef.cpp index 7cf0e2299ea..32a923cf890 100644 --- a/src/backend/gporca/libgpopt/src/base/CColRef.cpp +++ b/src/backend/gporca/libgpopt/src/base/CColRef.cpp @@ -107,11 +107,25 @@ IOstream & CColRef::OsPrint(IOstream &os) const { m_pname->OsPrint(os); - os << " (" << Id() << ")"; + os << " (" << Id() << "," << UsedStatusToString(m_used) << ")"; return os; } +const char* +CColRef::UsedStatusToString(CColRef::EUsedStatus status) const { + switch (status) { + case EUsed: return "Used"; + case EUnused: return "Unused"; + case EUnknown: return "Unknown"; + case ESentinel: return "Invalid"; + default: + return "Invalid"; + } + + return "Invalid"; +} + //--------------------------------------------------------------------------- // @function: // CColRef::Pdrgpul diff --git a/src/backend/gporca/libgpopt/src/base/COptimizationContext.cpp b/src/backend/gporca/libgpopt/src/base/COptimizationContext.cpp index 954dd96a8c1..c718a1c50f9 100644 --- a/src/backend/gporca/libgpopt/src/base/COptimizationContext.cpp +++ b/src/backend/gporca/libgpopt/src/base/COptimizationContext.cpp @@ -415,7 +415,7 @@ COptimizationContext::PrppCTEProducer(CMemoryPool *mp, UlongToColRefMap *colref_mapping = COptCtxt::PoctxtFromTLS()->Pcteinfo()->PhmulcrConsumerToProducer( mp, popProducer->UlCTEId(), pcrsInnerOutput, - popProducer->Pdrgpcr()); + popProducer->Pdrgpcr(), popProducer->PdrgpcrUnused()); CReqdPropPlan *prppProducer = CReqdPropPlan::PrppRemapForCTE( mp, pocProducer->Prpp(), pccProducer->Pdpplan(), pccConsumer->Pdpplan(), colref_mapping); diff --git a/src/backend/gporca/libgpopt/src/base/CReqdPropPlan.cpp b/src/backend/gporca/libgpopt/src/base/CReqdPropPlan.cpp index f0cfa78c6a5..177d409a2de 100644 --- a/src/backend/gporca/libgpopt/src/base/CReqdPropPlan.cpp +++ b/src/backend/gporca/libgpopt/src/base/CReqdPropPlan.cpp @@ -422,15 +422,13 @@ CReqdPropPlan::PrppEmpty(CMemoryPool *mp) IOstream & CReqdPropPlan::OsPrint(IOstream &os) const { - if (GPOS_FTRACE(EopttracePrintRequiredColumns)) + + os << "req cols: ["; + if (nullptr != m_pcrs) { - os << "req cols: ["; - if (nullptr != m_pcrs) - { - os << (*m_pcrs); - } - os << "], "; + os << (*m_pcrs); } + os << "], "; os << "req CTEs: ["; if (nullptr != m_pcter) diff --git a/src/backend/gporca/libgpopt/src/base/CUtils.cpp b/src/backend/gporca/libgpopt/src/base/CUtils.cpp index e41a04c5782..b5f847b817a 100644 --- a/src/backend/gporca/libgpopt/src/base/CUtils.cpp +++ b/src/backend/gporca/libgpopt/src/base/CUtils.cpp @@ -3657,7 +3657,9 @@ CUtils::PdrgpcrExactCopy(CMemoryPool *mp, CColRefArray *colref_array) // from old to copied variables are added to it. CColRefArray * CUtils::PdrgpcrCopy(CMemoryPool *mp, CColRefArray *colref_array, - BOOL fAllComputed, UlongToColRefMap *colref_mapping) + BOOL fAllComputed, UlongToColRefMap *colref_mapping, + UlongToColRefMap *consumer_mapping, UlongToColRefArrayMap *producer_mapping, + BOOL check_usage) { // get column factory from optimizer context object CColumnFactory *col_factory = COptCtxt::PoctxtFromTLS()->Pcf(); @@ -3678,13 +3680,63 @@ CUtils::PdrgpcrCopy(CMemoryPool *mp, CColRefArray *colref_array, new_colref = col_factory->PcrCopy(colref); } + if (check_usage) { + CColRef::EUsedStatus colref_status = colref->GetUsage(true, true); + switch(colref_status) { + case CColRef::EUsedStatus::EUsed: + { + new_colref->MarkAsUsed(); + break; + } + case CColRef::EUsedStatus::EUnused: + { + new_colref->MarkAsUnused(); + break; + } + case CColRef::EUsedStatus::EUnknown: + { + new_colref->MarkAsUnknown(); + break; + } + default: + GPOS_ASSERT(false); + } + } + pdrgpcrNew->Append(new_colref); + if (nullptr != colref_mapping) { BOOL fInserted GPOS_ASSERTS_ONLY = colref_mapping->Insert( GPOS_NEW(mp) ULONG(colref->Id()), new_colref); GPOS_ASSERT(fInserted); } + + if (nullptr != consumer_mapping) { + BOOL fInserted GPOS_ASSERTS_ONLY = consumer_mapping->Insert( + GPOS_NEW(mp) ULONG(new_colref->Id()), colref); + GPOS_ASSERT(fInserted); + } + + if (nullptr != producer_mapping) { + ULONG *producer_cid = GPOS_NEW(mp) ULONG(colref->Id()); + + const CColRefArray *crarray = producer_mapping->Find(producer_cid); + if (nullptr == crarray) + { + CColRefArray *crarray_new = GPOS_NEW(mp) CColRefArray(mp); + crarray_new->Append(new_colref); + + BOOL fInserted GPOS_ASSERTS_ONLY = + producer_mapping->Insert(producer_cid, crarray_new); + GPOS_ASSERT(fInserted); + } + else + { + (const_cast(crarray)) + ->Append(new_colref); + } + } } return pdrgpcrNew; diff --git a/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp b/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp index bf7accbbf53..d8d29700c0a 100644 --- a/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp @@ -320,10 +320,55 @@ CLogicalCTEConsumer::PopCopyWithRemappedColumns( CMemoryPool *mp, UlongToColRefMap *colref_mapping, BOOL must_exist) { CColRefArray *colref_array = nullptr; + CCTEInfo * cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); + if (must_exist) { + UlongToColRefMap *consumer_mapping; + UlongToColRefArrayMap *producer_mapping; + ULONG colid; + ULONG colref_size; + colref_array = CUtils::PdrgpcrRemapAndCreate(mp, m_pdrgpcr, colref_mapping); + + colref_size = colref_array->Size(); + GPOS_ASSERT(colref_size == colref_array->Size()); + + consumer_mapping = cteinfo->GetCTEConsumerMapping(); + producer_mapping = cteinfo->GetCTEProducerMapping(); + for (ULONG ul = 0; ul < colref_size; ul++) { + CColRef *old_colref = (*m_pdrgpcr)[ul]; + CColRef *new_colref = (*colref_array)[ul]; + + colid = old_colref->Id(); + CColRef *p_colref = consumer_mapping->Find(&colid); + if (nullptr == p_colref) { + GPOS_RAISE( + CException::ExmaInvalid, CException::ExmiInvalid, + GPOS_WSZ_LIT( + "Not found CTE consumer colid regsiterd in consumer mapping")); + } + + BOOL fInserted = consumer_mapping->Insert( + GPOS_NEW(mp) ULONG(new_colref->Id()), p_colref); + + if (fInserted) { + colid = p_colref->Id(); + + const CColRefArray *crarray = producer_mapping->Find(&colid); + if (nullptr == crarray) + { + GPOS_RAISE( + CException::ExmaInvalid, CException::ExmiInvalid, + GPOS_WSZ_LIT( + "Not found CTE consumer colid regsiterd in consumer mapping")); + } + + (const_cast(crarray)) + ->Append(new_colref); + } + } } else { diff --git a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp index ddd32ab723b..fd593ef523e 100644 --- a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp @@ -34,11 +34,21 @@ CPhysicalCTEConsumer::CPhysicalCTEConsumer(CMemoryPool *mp, ULONG id, UlongToColRefMap *colref_mapping) : CPhysical(mp), m_id(id), - m_pdrgpcr(colref_array), + m_pdrgpcr(nullptr), m_phmulcr(colref_mapping) { GPOS_ASSERT(nullptr != colref_array); GPOS_ASSERT(nullptr != colref_mapping); + + ULONG colref_size = colref_array->Size(); + m_pdrgpcr = GPOS_NEW(mp) CColRefArray(mp); + + for (ULONG index = 0; index < colref_size; index++) { + CColRef *col_ref = (*colref_array)[index]; + if (col_ref->GetUsage() == CColRef::EUsed) { + m_pdrgpcr->Append(col_ref); + } + } } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp index 95fe39e0260..d44877b4f54 100644 --- a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp @@ -31,10 +31,26 @@ using namespace gpopt; //--------------------------------------------------------------------------- CPhysicalCTEProducer::CPhysicalCTEProducer(CMemoryPool *mp, ULONG id, CColRefArray *colref_array) - : CPhysical(mp), m_id(id), m_pdrgpcr(colref_array), m_pcrs(nullptr) + : CPhysical(mp), m_id(id), m_pdrgpcr(nullptr), m_pcrs(nullptr) { GPOS_ASSERT(nullptr != colref_array); - m_pcrs = GPOS_NEW(mp) CColRefSet(mp, m_pdrgpcr); + ULONG colref_size = colref_array->Size(); + ULONG unused_inc = 0; + m_pcrs = GPOS_NEW(mp) CColRefSet(mp); + m_pdrgpcr = GPOS_NEW(mp) CColRefArray(mp); + m_pdrgpcr_unused = GPOS_NEW(mp) ULongPtrArray(mp, colref_size); + + for (ULONG index = 0; index < colref_size; index++) { + CColRef *col_ref = (*colref_array)[index]; + GPOS_ASSERT(col_ref->GetUsage() != CColRef::EUnknown); + m_pdrgpcr_unused->Append(GPOS_NEW(m_mp) ULONG(unused_inc)); + if (col_ref->GetUsage() == CColRef::EUsed) { + m_pdrgpcr->Append(col_ref); + m_pcrs->Include(col_ref); + } else { + unused_inc++; + } + } } //--------------------------------------------------------------------------- @@ -49,6 +65,7 @@ CPhysicalCTEProducer::~CPhysicalCTEProducer() { m_pdrgpcr->Release(); m_pcrs->Release(); + m_pdrgpcr_unused->Release(); } //--------------------------------------------------------------------------- @@ -69,12 +86,20 @@ CPhysicalCTEProducer::PcrsRequired(CMemoryPool *mp, CExpressionHandle &exprhdl, GPOS_ASSERT(0 == child_index); GPOS_ASSERT(0 == pcrsRequired->Size()); - CColRefSet *pcrs = GPOS_NEW(mp) CColRefSet(mp, *m_pcrs); + CColRefSet *pcrs = GPOS_NEW(mp) CColRefSet(mp); + ULONG ccr_size = m_pdrgpcr->Size(); + for (ULONG index = 0; index < ccr_size; index++) { + CColRef *col_ref = (*m_pdrgpcr)[index]; + GPOS_ASSERT(col_ref->GetUsage() != CColRef::EUnknown); + if (col_ref->GetUsage() == CColRef::EUsed) { + pcrs->Include(col_ref); + } + } + pcrs->Union(pcrsRequired); CColRefSet *pcrsChildReqd = PcrsChildReqd(mp, exprhdl, pcrs, child_index, gpos::ulong_max); - GPOS_ASSERT(pcrsChildReqd->Size() == m_pdrgpcr->Size()); pcrs->Release(); return pcrsChildReqd; diff --git a/src/backend/gporca/libgpopt/src/operators/CPhysicalTableScan.cpp b/src/backend/gporca/libgpopt/src/operators/CPhysicalTableScan.cpp index 1b553bfd48f..b083905d0ab 100644 --- a/src/backend/gporca/libgpopt/src/operators/CPhysicalTableScan.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CPhysicalTableScan.cpp @@ -100,7 +100,10 @@ CPhysicalTableScan::OsPrint(IOstream &os) const // actual name of table in catalog and columns os << " ("; m_ptabdesc->Name().OsPrint(os); - os << ")"; + os << "), Columns: ["; + + CUtils::OsPrintDrgPcr(os, m_pdrgpcrOutput); + os << "]"; return os; } diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp index bad4b1e8123..1d7851855c4 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp @@ -173,8 +173,6 @@ CTranslatorDXLToExpr::CTranslatorDXLToExpr(CMemoryPool *mp, { // initialize hash tables m_phmulcr = GPOS_NEW(m_mp) UlongToColRefMap(m_mp); - - // initialize hash tables m_phmululCTE = GPOS_NEW(m_mp) UlongToUlongMap(m_mp); if (fInitColumnFactory) @@ -278,7 +276,7 @@ CTranslatorDXLToExpr::Pexpr(const CDXLNode *dxlnode, const ULONG colid = dxl_colref->Id(); // get its column reference from the hash map - const CColRef *colref = LookupColRef(m_phmulcr, colid); + const CColRef *colref = LookupColRef(colid); if (fGenerateRequiredColumns) { @@ -721,8 +719,7 @@ CTranslatorDXLToExpr::PexprLogicalSetOp(const CDXLNode *dxlnode) PdrgpexprPreprocessSetOpInputs(dxlnode, pdrgdrgpcrInput, pdrgpulOutput); // create an array of output column references - CColRefArray *pdrgpcrOutput = CTranslatorDXLToExprUtils::Pdrgpcr( - m_mp, m_phmulcr, pdrgpulOutput /*array of colids of the first child*/); + CColRefArray *pdrgpcrOutput = Pdrgpcr(pdrgpulOutput /*array of colids of the first child*/); pdrgpulOutput->Release(); @@ -879,7 +876,7 @@ CTranslatorDXLToExpr::BuildSetOpChild( { // column identifier of the input column ULONG colid = *(*pdrgpulInput)[ulColPos]; - const CColRef *colref = LookupColRef(m_phmulcr, colid); + const CColRef *colref = LookupColRef(colid); // corresponding output column descriptor const CDXLColDescr *pdxlcdOutput = dxl_op->GetColumnDescrAt(ulColPos); @@ -1033,6 +1030,71 @@ CTranslatorDXLToExpr::FCastingUnknownType(IMDId *pmdidSource, IMDId *mdid_dest) mdid_dest->Equals(&CMDIdGPDB::m_mdid_unknown))); } +void +CTranslatorDXLToExpr::MarkCTEConsumerColAsUsed(UlongToColRefMap *mcidcrCTE, UlongToColRefArrayMap *mpidcrsCTE, ULONG colid) { + CColRef *producer_colref = nullptr; + producer_colref = mcidcrCTE->Find(&colid); + if (producer_colref) { + +#ifdef GPOS_DEBUG + if (GPOS_FTRACE(EopttraceDebugCTE)) { + CAutoTrace at(m_mp); + auto & io_stream = at.Os(); + io_stream << "MarkProducerUsed, customer id: " << colid << ", producer id:" << producer_colref->Id() << std::endl; + } +#endif + if (producer_colref->GetUsage(true, true) != CColRef::EUsed) { + producer_colref->MarkAsUsed(); + MarkCTEProducerColAsUsed(mcidcrCTE, mpidcrsCTE, producer_colref->Id()); + } + } +} + +void +CTranslatorDXLToExpr::MarkCTEProducerColAsUsed(UlongToColRefMap *mcidcrCTE, UlongToColRefArrayMap *mpidcrsCTE, ULONG colid) { + CColRefArray *consumer_crarray = nullptr; + +#ifdef GPOS_DEBUG + CAutoTrace at(m_mp); + auto & io_stream = at.Os(); + if (GPOS_FTRACE(EopttraceDebugCTE)) + io_stream << "MarkCTEProducerColAsUsed, producer id: " << colid; +#endif + + consumer_crarray = mpidcrsCTE->Find(&colid); + if (consumer_crarray) { + +#ifdef GPOS_DEBUG + if (GPOS_FTRACE(EopttraceDebugCTE)) + io_stream << ", consumer ids: ["; +#endif + + ULONG crarray_size = consumer_crarray->Size(); + for (ULONG crindex = 0; crindex < crarray_size; crindex++){ + auto consumer_colref = (*consumer_crarray)[crindex]; + +#ifdef GPOS_DEBUG + if (GPOS_FTRACE(EopttraceDebugCTE)) + io_stream << (*consumer_crarray)[crindex]->Id() << ","; +#endif + + if (consumer_colref->GetUsage(true, true) != CColRef::EUsed) { + consumer_colref->MarkAsUsed(); + MarkCTEConsumerColAsUsed(mcidcrCTE, mpidcrsCTE, consumer_colref->Id()); + } + } + +#ifdef GPOS_DEBUG + if (GPOS_FTRACE(EopttraceDebugCTE)) + io_stream << "]"; +#endif + } +#ifdef GPOS_DEBUG + else if (GPOS_FTRACE(EopttraceDebugCTE)) { + io_stream << ",not found consumer."; + } +#endif +} //--------------------------------------------------------------------------- // @function: @@ -1043,24 +1105,29 @@ CTranslatorDXLToExpr::FCastingUnknownType(IMDId *pmdidSource, IMDId *mdid_dest) // the column is not found //--------------------------------------------------------------------------- CColRef * -CTranslatorDXLToExpr::LookupColRef(UlongToColRefMap *colref_mapping, - ULONG colid) +CTranslatorDXLToExpr::LookupColRef(ULONG colid, BOOL mark_used) { - GPOS_ASSERT(nullptr != colref_mapping); + CColRef *colref = nullptr; GPOS_ASSERT(gpos::ulong_max != colid); - // get its column reference from the hash map - CColRef *colref = colref_mapping->Find(&colid); + colref = m_phmulcr->Find(&colid); if (nullptr == colref) { GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiDXL2ExprAttributeNotFound, colid); } - colref->MarkAsUsed(); + auto cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); + + if (cteinfo->ExistCTE()) { + MarkCTEConsumerColAsUsed(cteinfo->GetCTEConsumerMapping(), cteinfo->GetCTEProducerMapping(), colref->Id()); + MarkCTEProducerColAsUsed(cteinfo->GetCTEConsumerMapping(), cteinfo->GetCTEProducerMapping(), colref->Id()); + } + + if (mark_used) + colref->MarkAsUsed(); return colref; } - //--------------------------------------------------------------------------- // @function: // CTranslatorDXLToExpr::PcrCreate @@ -1120,6 +1187,35 @@ CTranslatorDXLToExpr::Pdrgpcr(const CDXLColDescrArray *dxl_col_descr_array) return pdrgpcrOutput; } +//--------------------------------------------------------------------------- +// @function: +// CTranslatorDXLToExpr::Pdrgpcr +// +// @doc: +// Construct a dynamic array of column references corresponding to the +// given col ids +// +//--------------------------------------------------------------------------- +CColRefArray * +CTranslatorDXLToExpr::Pdrgpcr(const ULongPtrArray *colids) +{ + GPOS_ASSERT(nullptr != colids); + + CColRefArray *colref_array = GPOS_NEW(m_mp) CColRefArray(m_mp); + + for (ULONG ul = 0; ul < colids->Size(); ul++) + { + ULONG *pulColId = (*colids)[ul]; + CColRef *colref = LookupColRef(*pulColId, true); + GPOS_ASSERT(nullptr != colref); + + colref_array->Append(colref); + } + + return colref_array; +} + + //--------------------------------------------------------------------------- // @function: // CTranslatorDXLToExpr::ConstructDXLColId2ColRefMapping @@ -1303,12 +1399,28 @@ CTranslatorDXLToExpr::PexprLogicalCTEProducer(const CDXLNode *dxlnode) ULongPtrArray *pdrgpulCols = pdxlopCTEProducer->GetOutputColIdsArray(); const ULONG length = pdrgpulCols->Size(); + +#ifdef GPOS_DEBUG + CAutoTrace at(m_mp); + auto & io_stream = at.Os(); + if (GPOS_FTRACE(EopttraceDebugCTE)) { + io_stream << "PexprLogicalCTEProducer, output col lengths: " << length << "" << std::endl; + } +#endif + for (ULONG ul = 0; ul < length; ul++) { ULONG *pulColId = (*pdrgpulCols)[ul]; - CColRef *colref = LookupColRef(m_phmulcr, *pulColId); + CColRef *colref = LookupColRef(*pulColId, !pdxlopCTEProducer->CouldBePruned() /* mark_used */); GPOS_ASSERT(nullptr != colref); +#ifdef GPOS_DEBUG + if (GPOS_FTRACE(EopttraceDebugCTE)) { + io_stream << "DXL id: " << *pulColId << ", col: "; + colref->OsPrint(io_stream); + io_stream << std::endl; + } +#endif if (pcrsProducer->FMember(colref)) { // the column was previously used, so introduce a project node to relabel @@ -1371,9 +1483,11 @@ CTranslatorDXLToExpr::PexprLogicalCTEConsumer(const CDXLNode *dxlnode) CExpression *pexprProducer = pcteinfo->PexprCTEProducer(id); GPOS_ASSERT(nullptr != pexprProducer); + CCTEInfo * cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); CColRefArray *pdrgpcrProducer = CLogicalCTEProducer::PopConvert(pexprProducer->Pop())->Pdrgpcr(); - CColRefArray *pdrgpcrConsumer = CUtils::PdrgpcrCopy(m_mp, pdrgpcrProducer); + CColRefArray *pdrgpcrConsumer = CUtils::PdrgpcrCopy(m_mp, pdrgpcrProducer, + false, nullptr, cteinfo->GetCTEConsumerMapping(), cteinfo->GetCTEProducerMapping(), true); // add new colrefs to mapping const ULONG num_cols = pdrgpcrConsumer->Size(); @@ -1385,6 +1499,26 @@ CTranslatorDXLToExpr::PexprLogicalCTEConsumer(const CDXLNode *dxlnode) BOOL result GPOS_ASSERTS_ONLY = m_phmulcr->Insert(pulColId, colref); GPOS_ASSERT(result); + +#ifdef GPOS_DEBUG + CAutoTrace at(m_mp); + auto & io_stream = at.Os(); + if (GPOS_FTRACE(EopttraceDebugCTE)) { + + io_stream << "m_phmulcr->Insert: " << *pulColId << ","; + colref->OsPrint(io_stream); + io_stream << std::endl; + io_stream << "PexprLogicalCTEConsumer m_phmulcr: " << std::endl; + UlongToColRefMapIter iter(m_phmulcr); + while (iter.Advance()) + { + CColRef *colref = m_phmulcr->Find(iter.Key()); + io_stream << "key: " << *iter.Key() << ", value: "; + colref->OsPrint(io_stream); + io_stream << " ."; + } + } +#endif } pcteinfo->IncrementConsumers(id, m_ulCTEId); @@ -1440,8 +1574,7 @@ CTranslatorDXLToExpr::PexprLogicalInsert(const CDXLNode *dxlnode) CTableDescriptor *ptabdesc = Ptabdesc(pdxlopInsert->GetDXLTableDescr()); ULongPtrArray *pdrgpulSourceCols = pdxlopInsert->GetSrcColIdsArray(); - CColRefArray *colref_array = - CTranslatorDXLToExprUtils::Pdrgpcr(m_mp, m_phmulcr, pdrgpulSourceCols); + CColRefArray *colref_array = Pdrgpcr(pdrgpulSourceCols); return GPOS_NEW(m_mp) CExpression( m_mp, GPOS_NEW(m_mp) CLogicalInsert(m_mp, ptabdesc, colref_array), @@ -1478,12 +1611,11 @@ CTranslatorDXLToExpr::PexprLogicalDelete(const CDXLNode *dxlnode) ULONG ctid_colid = pdxlopDelete->GetCtIdColId(); ULONG segid_colid = pdxlopDelete->GetSegmentIdColId(); - CColRef *pcrCtid = LookupColRef(m_phmulcr, ctid_colid); - CColRef *pcrSegmentId = LookupColRef(m_phmulcr, segid_colid); + CColRef *pcrCtid = LookupColRef(ctid_colid); + CColRef *pcrSegmentId = LookupColRef(segid_colid); ULongPtrArray *pdrgpulCols = pdxlopDelete->GetDeletionColIdArray(); - CColRefArray *colref_array = - CTranslatorDXLToExprUtils::Pdrgpcr(m_mp, m_phmulcr, pdrgpulCols); + CColRefArray *colref_array = Pdrgpcr(pdrgpulCols); return GPOS_NEW(m_mp) CExpression( m_mp, @@ -1542,16 +1674,16 @@ CTranslatorDXLToExpr::PexprLogicalUpdate(const CDXLNode *dxlnode) ULONG ctid_colid = pdxlopUpdate->GetCtIdColId(); ULONG segid_colid = pdxlopUpdate->GetSegmentIdColId(); - CColRef *pcrCtid = LookupColRef(m_phmulcr, ctid_colid); - CColRef *pcrSegmentId = LookupColRef(m_phmulcr, segid_colid); + CColRef *pcrCtid = LookupColRef(ctid_colid); + CColRef *pcrSegmentId = LookupColRef(segid_colid); ULongPtrArray *pdrgpulInsertCols = pdxlopUpdate->GetInsertionColIdArray(); CColRefArray *pdrgpcrInsert = - CTranslatorDXLToExprUtils::Pdrgpcr(m_mp, m_phmulcr, pdrgpulInsertCols); + Pdrgpcr(pdrgpulInsertCols); ULongPtrArray *pdrgpulDeleteCols = pdxlopUpdate->GetDeletionColIdArray(); CColRefArray *pdrgpcrDelete = - CTranslatorDXLToExprUtils::Pdrgpcr(m_mp, m_phmulcr, pdrgpulDeleteCols); + Pdrgpcr(pdrgpulDeleteCols); return GPOS_NEW(m_mp) CExpression(m_mp, @@ -1583,8 +1715,7 @@ CTranslatorDXLToExpr::PexprLogicalCTAS(const CDXLNode *dxlnode) CTableDescriptor *ptabdesc = PtabdescFromCTAS(pdxlopCTAS); ULongPtrArray *pdrgpulSourceCols = pdxlopCTAS->GetSrcColidsArray(); - CColRefArray *colref_array = - CTranslatorDXLToExprUtils::Pdrgpcr(m_mp, m_phmulcr, pdrgpulSourceCols); + CColRefArray *colref_array = Pdrgpcr(pdrgpulSourceCols); return GPOS_NEW(m_mp) CExpression( m_mp, GPOS_NEW(m_mp) CLogicalInsert(m_mp, ptabdesc, colref_array), @@ -1616,8 +1747,7 @@ CTranslatorDXLToExpr::PexprLogicalGroupBy(const CDXLNode *dxlnode) CExpression *pexprProjList = PexprScalarProjList(pdxlnPrL); // translate grouping columns - CColRefArray *pdrgpcrGroupingCols = CTranslatorDXLToExprUtils::Pdrgpcr( - m_mp, m_phmulcr, pdxlopGrpby->GetGroupingColidArray()); + CColRefArray *pdrgpcrGroupingCols = Pdrgpcr(pdxlopGrpby->GetGroupingColidArray()); if (0 != pexprProjList->Arity()) { @@ -1908,7 +2038,7 @@ CTranslatorDXLToExpr::PdrgpcrPartitionByCol( const ULONG *pulColId = (*partition_by_colid_array)[ul]; // get its column reference from the hash map - CColRef *colref = LookupColRef(m_phmulcr, *pulColId); + CColRef *colref = LookupColRef(*pulColId); colref_array->Append(colref); } @@ -2533,7 +2663,7 @@ CTranslatorDXLToExpr::PexprScalarSubqueryQuantified( CExpression *pexprScalarChild = Pexpr(pdxlnScalarChild); // get colref for subquery colid - const CColRef *colref = LookupColRef(m_phmulcr, colid); + const CColRef *colref = LookupColRef(colid); CScalar *popScalarSubquery = nullptr; if (EdxlopScalarSubqueryAny == edxlopid) @@ -3490,7 +3620,7 @@ CTranslatorDXLToExpr::PexprScalarIdent(const CDXLNode *pdxlnIdent) const ULONG colid = dxl_colref->Id(); // get its column reference from the hash map - const CColRef *colref = LookupColRef(m_phmulcr, colid); + const CColRef *colref = LookupColRef(colid); CExpression *pexpr = GPOS_NEW(m_mp) CExpression(m_mp, GPOS_NEW(m_mp) CScalarIdent(m_mp, colref)); @@ -3955,7 +4085,7 @@ CTranslatorDXLToExpr::PexprScalarSubquery(const CDXLNode *pdxlnSubquery) // get subquery colref for colid ULONG colid = pdxlopSubquery->GetColId(); - const CColRef *colref = LookupColRef(m_phmulcr, colid); + const CColRef *colref = LookupColRef(colid); CScalarSubquery *popScalarSubquery = GPOS_NEW(m_mp) CScalarSubquery(m_mp, colref, false /*fGeneratedByExist*/, @@ -4080,7 +4210,7 @@ CTranslatorDXLToExpr::Pos(const CDXLNode *dxlnode) const ULONG colid = dxl_op->GetColId(); // get its column reference from the hash map - CColRef *colref = LookupColRef(m_phmulcr, colid); + CColRef *colref = LookupColRef(colid); IMDId *sort_op_id = dxl_op->GetMdIdSortOp(); sort_op_id->AddRef(); diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExprUtils.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExprUtils.cpp index ef35ace4a97..b487c979761 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExprUtils.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExprUtils.cpp @@ -213,37 +213,6 @@ CTranslatorDXLToExprUtils::EBoolOperator(EdxlBoolExprType edxlbooltype) return eboolop; } -//--------------------------------------------------------------------------- -// @function: -// CTranslatorDXLToExprUtils::Pdrgpcr -// -// @doc: -// Construct a dynamic array of column references corresponding to the -// given col ids -// -//--------------------------------------------------------------------------- -CColRefArray * -CTranslatorDXLToExprUtils::Pdrgpcr(CMemoryPool *mp, - UlongToColRefMap *colref_mapping, - const ULongPtrArray *colids) -{ - GPOS_ASSERT(nullptr != colids); - - CColRefArray *colref_array = GPOS_NEW(mp) CColRefArray(mp); - - for (ULONG ul = 0; ul < colids->Size(); ul++) - { - ULONG *pulColId = (*colids)[ul]; - CColRef *colref = colref_mapping->Find(pulColId); - colref->MarkAsUsed(); - GPOS_ASSERT(nullptr != colref); - - colref_array->Append(colref); - } - - return colref_array; -} - //--------------------------------------------------------------------------- // @function: diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLLogicalCTEProducer.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLLogicalCTEProducer.h index 6ce5567dbb7..04611ac2258 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLLogicalCTEProducer.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLLogicalCTEProducer.h @@ -34,12 +34,14 @@ class CDXLLogicalCTEProducer : public CDXLLogical // output column ids ULongPtrArray *m_output_colids_array; + BOOL m_could_be_pruned; + public: CDXLLogicalCTEProducer(CDXLLogicalCTEProducer &) = delete; // ctor CDXLLogicalCTEProducer(CMemoryPool *mp, ULONG id, - ULongPtrArray *output_colids_array); + ULongPtrArray *output_colids_array, BOOL could_be_pruned); // dtor ~CDXLLogicalCTEProducer() override; @@ -63,6 +65,12 @@ class CDXLLogicalCTEProducer : public CDXLLogical return m_output_colids_array; } + BOOL + CouldBePruned() const + { + return m_could_be_pruned; + } + // serialize operator in DXL format void SerializeToDXL(CXMLSerializer *xml_serializer, const CDXLNode *dxlnode) const override; diff --git a/src/backend/gporca/libnaucrates/include/naucrates/statistics/IStatistics.h b/src/backend/gporca/libnaucrates/include/naucrates/statistics/IStatistics.h index c5d4ab38957..25682a21399 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/statistics/IStatistics.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/statistics/IStatistics.h @@ -58,10 +58,16 @@ using UlongToDoubleMapIter = CHashMapIter, gpos::Equals, CleanupDelete, CleanupDelete>; +// hash map from column ULONG to ULONG using UlongToUlongMap = CHashMap, gpos::Equals, CleanupDelete, CleanupDelete>; +// iterator +using UlongToUlongMapIter = + CHashMapIter, gpos::Equals, + CleanupDelete, CleanupDelete>; + // hash maps mapping INT -> ULONG using IntToUlongMap = CHashMap, gpos::Equals, diff --git a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h index b4aea06ac8f..684fcd60b26 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h @@ -71,15 +71,15 @@ enum EOptTraceFlag // print MEMO during property enforcement process EopttracePrintMemoEnforcement = 101015, - // print required columns - EopttracePrintRequiredColumns = 101016, - // print equivalent distribution specs EopttracePrintEquivDistrSpecs = 101017, // log results of hint parsing EopttracePrintPgHintPlanLog = 101018, + // print debug info of CTE + EopttraceDebugCTE = 101019, + /////////////////////////////////////////////////////// ////////////////// transformations flags ////////////// /////////////////////////////////////////////////////// @@ -274,6 +274,8 @@ enum EOptTraceFlag // is nestloop params enabled, it is only enabled in GPDB 6.x onwards. EopttraceIndexedNLJOuterRefAsParams = 106000, + // + // max EopttraceSentinel = 199999 }; diff --git a/src/backend/gporca/libnaucrates/src/operators/CDXLLogicalCTEProducer.cpp b/src/backend/gporca/libnaucrates/src/operators/CDXLLogicalCTEProducer.cpp index 9b8ef4f79a6..5dac22f2ad3 100644 --- a/src/backend/gporca/libnaucrates/src/operators/CDXLLogicalCTEProducer.cpp +++ b/src/backend/gporca/libnaucrates/src/operators/CDXLLogicalCTEProducer.cpp @@ -31,8 +31,8 @@ using namespace gpdxl; // //--------------------------------------------------------------------------- CDXLLogicalCTEProducer::CDXLLogicalCTEProducer( - CMemoryPool *mp, ULONG id, ULongPtrArray *output_colids_array) - : CDXLLogical(mp), m_id(id), m_output_colids_array(output_colids_array) + CMemoryPool *mp, ULONG id, ULongPtrArray *output_colids_array, BOOL could_be_pruned) + : CDXLLogical(mp), m_id(id), m_output_colids_array(output_colids_array), m_could_be_pruned(could_be_pruned) { GPOS_ASSERT(nullptr != output_colids_array); } diff --git a/src/backend/gporca/libnaucrates/src/parser/CParseHandlerLogicalCTEProducer.cpp b/src/backend/gporca/libnaucrates/src/parser/CParseHandlerLogicalCTEProducer.cpp index b6a6cd85c36..8d74ae54c78 100644 --- a/src/backend/gporca/libnaucrates/src/parser/CParseHandlerLogicalCTEProducer.cpp +++ b/src/backend/gporca/libnaucrates/src/parser/CParseHandlerLogicalCTEProducer.cpp @@ -72,7 +72,7 @@ CParseHandlerLogicalCTEProducer::StartElement( m_dxl_node = GPOS_NEW(m_mp) CDXLNode( m_mp, - GPOS_NEW(m_mp) CDXLLogicalCTEProducer(m_mp, id, output_colids_array)); + GPOS_NEW(m_mp) CDXLLogicalCTEProducer(m_mp, id, output_colids_array, true /*could_be_pruned*/)); // create and activate the parse handler for the child expression node CParseHandlerBase *child_parse_handler = diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index e342d762705..a8cbe740829 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -311,6 +311,7 @@ bool optimizer_print_group_properties; bool optimizer_print_optimization_context; bool optimizer_print_optimization_stats; bool optimizer_print_xform_results; +bool optimizer_debug_cte; /* array of xforms disable flags */ bool optimizer_xforms[OPTIMIZER_XFORMS_COUNT] = {[0 ... OPTIMIZER_XFORMS_COUNT - 1] = false}; @@ -2011,6 +2012,17 @@ struct config_bool ConfigureNamesBool_gp[] = NULL, NULL, NULL }, + { + {"optimizer_debug_cte", PGC_USERSET, LOGGING_WHAT, + gettext_noop("Print the debug info of CTE in ORCA."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &optimizer_debug_cte, + false, + NULL, NULL, NULL + }, + { {"optimizer_print_memo_after_exploration", PGC_USERSET, LOGGING_WHAT, gettext_noop("Print optimizer memo structure after the exploration phase."), diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 63a69864d9c..5f7df79ba6e 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -489,6 +489,7 @@ extern bool optimizer_print_group_properties; extern bool optimizer_print_optimization_context; extern bool optimizer_print_optimization_stats; extern bool optimizer_print_xform_results; +extern bool optimizer_debug_cte; /* array of xforms disable flags */ extern bool optimizer_xforms[OPTIMIZER_XFORMS_COUNT]; diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index 6cbf4b3179b..eb0c889c39d 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -475,6 +475,7 @@ "optimizer_print_query", "optimizer_print_xform", "optimizer_print_xform_results", + "optimizer_debug_cte", "optimizer_prune_computed_columns", "optimizer_push_group_by_below_setop_threshold", "optimizer_push_requirements_from_consumer_to_producer", diff --git a/src/test/regress/expected/cte_prune.out b/src/test/regress/expected/cte_prune.out new file mode 100644 index 00000000000..4534dadc4d9 --- /dev/null +++ b/src/test/regress/expected/cte_prune.out @@ -0,0 +1,1198 @@ +-- start_ignore +drop table if exists t1; +drop table if exists t2; +-- end_ignore +create table t1(v1 int, v2 int, v3 int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t1 values(generate_series(1, 10), generate_series(11, 20), generate_series(21, 30)); +analyze t1; +create table t2(v1 int, v2 int, v3 int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t2 values(generate_series(0, 100), generate_series(100, 200), generate_series(200, 300)); +-- should pruned both seq scan and shared scan +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +--------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) + Output: c11.v1 + -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) + Output: c11.v1 + Hash Cond: (t1.v1 = c11.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.04..1.04 rows=1 width=4) + Output: c11.v1 + -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=4) + Output: c11.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Filter: (t1_1.v1 < 5) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v1 +---- + 1 + 2 + 3 + 4 +(4 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +--------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) + Output: c11.v2 + -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) + Output: c11.v2 + Hash Cond: (t1.v1 = c11.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.04..1.04 rows=1 width=8) + Output: c11.v2, c11.v1 + -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=8) + Output: c11.v2, c11.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Filter: (t1_1.v1 < 5) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v2 +---- + 12 + 13 + 14 + 11 +(4 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +--------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) + Output: c11.v3 + -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) + Output: c11.v3 + Hash Cond: (t1.v1 = c11.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.04..1.04 rows=1 width=8) + Output: c11.v3, c11.v1 + -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=8) + Output: c11.v3, c11.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Filter: (t1_1.v1 < 5) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v3 +---- + 22 + 23 + 24 + 21 +(4 rows) + +-- * also should be pruned +explain verbose with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +--------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) + Output: c11.v1 + -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) + Output: c11.v1 + Hash Cond: (t1.v1 = c11.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.04..1.04 rows=1 width=4) + Output: c11.v1 + -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=4) + Output: c11.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Filter: (t1_1.v1 < 5) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v1 +---- + 1 + 2 + 3 + 4 +(4 rows) + +-- no push filter +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + QUERY PLAN +---------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.08..2.35 rows=10 width=4) + Output: c11.v3 + -> Hash Right Join (cost=1.08..2.22 rows=3 width=4) + Output: c11.v3 + Hash Cond: (c22.v2 = c11.v1) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1.10 rows=3 width=4) + Output: c22.v2 + Hash Key: c22.v2 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v2 + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=8) + Output: c11.v3, c11.v1 + -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=8) + Output: c11.v3, c11.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(20 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + v3 +---- + 26 + 25 + 30 + 29 + 21 + 27 + 23 + 28 + 24 + 22 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + QUERY PLAN +---------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.08..2.35 rows=10 width=4) + Output: c11.v2 + -> Hash Right Join (cost=1.08..2.22 rows=3 width=4) + Output: c11.v2 + Hash Cond: (c22.v2 = c11.v1) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1.10 rows=3 width=4) + Output: c22.v2 + Hash Key: c22.v2 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v2 + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=8) + Output: c11.v2, c11.v1 + -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=8) + Output: c11.v2, c11.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(20 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + v2 +---- + 17 + 13 + 18 + 14 + 12 + 16 + 15 + 20 + 19 + 11 +(10 rows) + +-- distribution col can't pruned +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.14..2.42 rows=10 width=4) + Output: c11.v2 + -> Hash Left Join (cost=1.14..2.29 rows=3 width=4) + Output: c11.v2 + Hash Cond: (c11.v2 = c22.v2) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1.10 rows=3 width=4) + Output: c11.v2 + Hash Key: c11.v2 + -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=4) + Output: c11.v2 + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.10..1.10 rows=3 width=4) + Output: c22.v2 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..1.10 rows=3 width=4) + Output: c22.v2 + Hash Key: c22.v2 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v2 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(23 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; + v2 +---- + 18 + 16 + 19 + 13 + 14 + 17 + 11 + 12 + 15 + 20 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.14..2.42 rows=10 width=4) + Output: c11.v3 + -> Hash Left Join (cost=1.14..2.29 rows=3 width=4) + Output: c11.v3 + Hash Cond: (c11.v3 = c22.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1.10 rows=3 width=4) + Output: c11.v3 + Hash Key: c11.v3 + -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=4) + Output: c11.v3 + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.10..1.10 rows=3 width=4) + Output: c22.v3 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..1.10 rows=3 width=4) + Output: c22.v3 + Hash Key: c22.v3 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v3 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(23 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; + v3 +---- + 22 + 24 + 27 + 29 + 28 + 21 + 25 + 23 + 26 + 30 +(10 rows) + +-- groupby/order by/window function/grouping set should be contains in CTE output +-- group by +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.17..2.34 rows=10 width=12) + Output: (sum(t1.v1)), t1.v1 + -> HashAggregate (cost=2.17..2.20 rows=3 width=12) + Output: sum(t1.v1), t1.v1 + Group Key: t1.v1 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(18 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; + sum +----- + 4 + 2 + 8 + 7 + 3 + 10 + 9 + 6 + 5 + 1 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.24..2.40 rows=10 width=12) + Output: (sum(t1.v1)), t1.v2 + -> HashAggregate (cost=2.24..2.27 rows=3 width=12) + Output: sum(t1.v1), t1.v2 + Group Key: t1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.08..2.22 rows=3 width=8) + Output: t1.v2, t1.v1 + Hash Key: t1.v2 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v2, t1.v1 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(21 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + sum +----- + 6 + 9 + 8 + 10 + 2 + 5 + 1 + 3 + 4 + 7 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.24..2.40 rows=10 width=12) + Output: (sum(t1.v3)), t1.v2 + -> HashAggregate (cost=2.24..2.27 rows=3 width=12) + Output: sum(t1.v3), t1.v2 + Group Key: t1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.08..2.22 rows=3 width=8) + Output: t1.v2, t1.v3 + Hash Key: t1.v2 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v2, t1.v3 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(21 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + sum +----- + 26 + 29 + 28 + 30 + 22 + 25 + 21 + 23 + 24 + 27 +(10 rows) + +-- order by +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) + Output: t1.v1, c22.v1 + Merge Key: c22.v1 + -> Sort (cost=2.18..2.19 rows=3 width=8) + Output: t1.v1, c22.v1 + Sort Key: c22.v1 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v1, c22.v1 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(19 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; + v1 +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) + Output: t1.v1, c22.v3 + Merge Key: c22.v3 + -> Sort (cost=2.18..2.19 rows=3 width=8) + Output: t1.v1, c22.v3 + Sort Key: c22.v3 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v1, c22.v3 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=8) + Output: c22.v3, c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=8) + Output: c22.v3, c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(19 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; + v1 +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +(10 rows) + +-- window function +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + WindowAgg (cost=2.18..2.47 rows=10 width=12) + Output: sum(t1.v1) OVER (?), t1.v2 + Order By: t1.v2 + -> Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) + Output: t1.v2, t1.v1 + Merge Key: t1.v2 + -> Sort (cost=2.18..2.19 rows=3 width=8) + Output: t1.v2, t1.v1 + Sort Key: t1.v2 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v2, t1.v1 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(22 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + sum +----- + 1 + 3 + 6 + 10 + 15 + 21 + 28 + 36 + 45 + 55 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + WindowAgg (cost=2.18..2.47 rows=10 width=12) + Output: sum(t1.v2) OVER (?), t1.v3 + Order By: t1.v3 + -> Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) + Output: t1.v3, t1.v2 + Merge Key: t1.v3 + -> Sort (cost=2.18..2.19 rows=3 width=8) + Output: t1.v3, t1.v2 + Sort Key: t1.v3 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v3, t1.v2 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(22 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + sum +----- + 11 + 23 + 36 + 50 + 65 + 81 + 98 + 116 + 135 + 155 +(10 rows) + +-- grouping set +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v1,c11.v2); + QUERY PLAN +--------------------------------------------------------------------------------------------------- + GroupAggregate (cost=2.18..2.61 rows=21 width=16) + Output: sum(t1.v2), t1.v1, t1.v2 + Group Key: t1.v1, t1.v2 + Group Key: t1.v1 + Group Key: () + -> Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) + Output: t1.v1, t1.v2 + Merge Key: t1.v1, t1.v2 + -> Sort (cost=2.18..2.19 rows=3 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v1, t1.v2 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(24 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v1,c11.v2); + sum +----- + 11 + 11 + 12 + 12 + 13 + 13 + 14 + 14 + 15 + 15 + 16 + 16 + 17 + 17 + 18 + 18 + 19 + 19 + 20 + 20 + 155 +(21 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v2,c11.v3); + QUERY PLAN +--------------------------------------------------------------------------------------------------- + GroupAggregate (cost=2.18..2.61 rows=21 width=16) + Output: sum(t1.v2), t1.v2, t1.v3 + Group Key: t1.v2, t1.v3 + Group Key: t1.v2 + Group Key: () + -> Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) + Output: t1.v2, t1.v3 + Merge Key: t1.v2, t1.v3 + -> Sort (cost=2.18..2.19 rows=3 width=8) + Output: t1.v2, t1.v3 + Sort Key: t1.v2, t1.v3 + -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) + Output: t1.v2, t1.v3 + Hash Cond: (t1.v1 = c22.v1) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) + Output: c22.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v1, t1_1.v2, t1_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(24 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + sum +----- + 11 + 23 + 36 + 50 + 65 + 81 + 98 + 116 + 135 + 155 +(10 rows) + +-- CTE producer should have right output +explain verbose with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +--------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=366.90..742.37 rows=779 width=4) + Output: t1.v1 + -> Hash Left Join (cost=366.90..731.98 rows=260 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = c22.v1) + -> Hash Join (cost=1.08..362.58 rows=260 width=16) + Output: t1.v1, t2.v1, t2.v2, t2.v3 + Hash Cond: (t2.v1 = t1.v1) + -> Seq Scan on public.t2 (cost=0.00..293.67 rows=25967 width=12) + Output: t2.v1, t2.v2, t2.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: t1.v1 + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=4) + Output: t1.v1 + -> Hash (cost=362.58..362.58 rows=260 width=4) + Output: c22.v1 + -> Subquery Scan on c22 (cost=1.08..362.58 rows=260 width=4) + Output: c22.v1 + -> Hash Join (cost=1.08..362.58 rows=260 width=16) + Output: t1_1.v1, t2_1.v1, t2_1.v2, t2_1.v3 + Hash Cond: (t2_1.v1 = t1_1.v1) + -> Seq Scan on public.t2 t2_1 (cost=0.00..293.67 rows=25967 width=12) + Output: t2_1.v1, t2_1.v2, t2_1.v3 + -> Hash (cost=1.03..1.03 rows=3 width=4) + Output: t1_1.v1 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=4) + Output: t1_1.v1 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(29 rows) + +with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + v1 +---- + 5 + 6 + 9 + 10 + 2 + 3 + 4 + 7 + 8 + 1 +(10 rows) + +explain verbose with c1 as (select sum(v1) as v1, sum(v2) as v2, v3 from t1 group by v3) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.39..2.67 rows=10 width=8) + Output: c11.v1 + -> Hash Left Join (cost=2.39..2.54 rows=3 width=8) + Output: c11.v1 + Hash Cond: (c11.v1 = c22.v1) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.12..1.23 rows=3 width=8) + Output: c11.v1 + Hash Key: c11.v1 + -> Subquery Scan on c11 (cost=1.12..1.16 rows=3 width=8) + Output: c11.v1 + -> HashAggregate (cost=1.12..1.16 rows=3 width=20) + Output: sum(t1.v1), sum(t1.v2), t1.v3 + Group Key: t1.v3 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..1.10 rows=3 width=12) + Output: t1.v3, t1.v1, t1.v2 + Hash Key: t1.v3 + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v3, t1.v1, t1.v2 + -> Hash (cost=1.23..1.23 rows=3 width=8) + Output: c22.v1 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=1.12..1.23 rows=3 width=8) + Output: c22.v1 + Hash Key: c22.v1 + -> Subquery Scan on c22 (cost=1.12..1.16 rows=3 width=8) + Output: c22.v1 + -> HashAggregate (cost=1.12..1.16 rows=3 width=20) + Output: sum(t1_1.v1), sum(t1_1.v2), t1_1.v3 + Group Key: t1_1.v3 + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..1.10 rows=3 width=12) + Output: t1_1.v3, t1_1.v1, t1_1.v2 + Hash Key: t1_1.v3 + -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + Output: t1_1.v3, t1_1.v1, t1_1.v2 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(35 rows) + +with c1 as (select lt1.v3 as v3, lt1.v1 as lo1, rt1.v1 as ro1 from t1 lt1, t1 rt1 where lt1.v2 = rt1.v2 and lt1.v1 = rt1.v1) +select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 from c1 where v3 > 0); + v1 | v2 | v3 +----+----+---- +(0 rows) + +-- cte in cte +-- function call +-- TPCDS cte not support reduce producter output yet +-- start_ignore +drop table if exists tpcds_store_sales; +drop table if exists tpcds_date_dim; +drop table if exists tpcds_item; +drop table if exists tpcds_web_sales; +-- end_ignore +create table tpcds_store_sales +( + ss_sold_date_sk integer , + ss_sold_time_sk integer , + ss_item_sk integer not null, + ss_customer_sk integer , + ss_cdemo_sk integer , + ss_hdemo_sk integer , + ss_addr_sk integer , + ss_store_sk integer , + ss_promo_sk integer , + ss_ticket_number integer not null, + ss_quantity integer , + ss_wholesale_cost decimal(7,2) , + ss_list_price decimal(7,2) , + ss_sales_price decimal(7,2) , + ss_ext_discount_amt decimal(7,2) , + ss_ext_sales_price decimal(7,2) , + ss_ext_wholesale_cost decimal(7,2) , + ss_ext_list_price decimal(7,2) , + ss_ext_tax decimal(7,2) , + ss_coupon_amt decimal(7,2) , + ss_net_paid decimal(7,2) , + ss_net_paid_inc_tax decimal(7,2) , + ss_net_profit decimal(7,2) , + primary key (ss_item_sk, ss_ticket_number) +); +create table tpcds_date_dim +( + d_date_sk integer not null, + d_date_id char(16) not null, + d_date date , + d_month_seq integer , + d_week_seq integer , + d_quarter_seq integer , + d_year integer , + d_dow integer , + d_moy integer , + d_dom integer , + d_qoy integer , + d_fy_year integer , + d_fy_quarter_seq integer , + d_fy_week_seq integer , + d_day_name char(9) , + d_quarter_name char(6) , + d_holiday char(1) , + d_weekend char(1) , + d_following_holiday char(1) , + d_first_dom integer , + d_last_dom integer , + d_same_day_ly integer , + d_same_day_lq integer , + d_current_day char(1) , + d_current_week char(1) , + d_current_month char(1) , + d_current_quarter char(1) , + d_current_year char(1) , + primary key (d_date_sk) +); +create table tpcds_item +( + i_item_sk integer not null, + i_item_id char(16) not null, + i_rec_start_date date , + i_rec_end_date date , + i_item_desc varchar(200) , + i_current_price decimal(7,2) , + i_wholesale_cost decimal(7,2) , + i_brand_id integer , + i_brand char(50) , + i_class_id integer , + i_class char(50) , + i_category_id integer , + i_category char(50) , + i_manufact_id integer , + i_manufact char(50) , + i_size char(20) , + i_formulation char(20) , + i_color char(20) , + i_units char(10) , + i_container char(10) , + i_manager_id integer , + i_product_name char(50) , + primary key (i_item_sk) +); +create table tpcds_web_sales +( + ws_sold_date_sk integer , + ws_sold_time_sk integer , + ws_ship_date_sk integer , + ws_item_sk integer not null, + ws_bill_customer_sk integer , + ws_bill_cdemo_sk integer , + ws_bill_hdemo_sk integer , + ws_bill_addr_sk integer , + ws_ship_customer_sk integer , + ws_ship_cdemo_sk integer , + ws_ship_hdemo_sk integer , + ws_ship_addr_sk integer , + ws_web_page_sk integer , + ws_web_site_sk integer , + ws_ship_mode_sk integer , + ws_warehouse_sk integer , + ws_promo_sk integer , + ws_order_number integer not null, + ws_quantity integer , + ws_wholesale_cost decimal(7,2) , + ws_list_price decimal(7,2) , + ws_sales_price decimal(7,2) , + ws_ext_discount_amt decimal(7,2) , + ws_ext_sales_price decimal(7,2) , + ws_ext_wholesale_cost decimal(7,2) , + ws_ext_list_price decimal(7,2) , + ws_ext_tax decimal(7,2) , + ws_coupon_amt decimal(7,2) , + ws_ext_ship_cost decimal(7,2) , + ws_net_paid decimal(7,2) , + ws_net_paid_inc_tax decimal(7,2) , + ws_net_paid_inc_ship decimal(7,2) , + ws_net_paid_inc_ship_tax decimal(7,2) , + ws_net_profit decimal(7,2) , +primary key (ws_item_sk, ws_order_number) +); +-- sql 23 +explain verbose with frequent_ss_items as + (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + from tpcds_store_sales + ,tpcds_date_dim + ,tpcds_item + where ss_sold_date_sk = d_date_sk + and ss_item_sk = i_item_sk + and d_year in (1999,1999+1,1999+2,1999+3) + group by substr(i_item_desc,1,30),i_item_sk,d_date + having count(*) >4) +select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) + and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=452.38..452.44 rows=3 width=4) + Output: t1.v1 + -> HashAggregate (cost=452.38..452.39 rows=1 width=4) + Output: t1.v1 + Group Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=451.80..452.38 rows=1 width=4) + Output: t1.v1, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=451.80..452.36 rows=1 width=4) + Output: t1.v1, (RowIdExpr) + Hash Cond: (frequent_ss_items.item_sk = t1.v1) + -> Subquery Scan on frequent_ss_items (cost=223.67..224.14 rows=6 width=4) + Output: frequent_ss_items.item_sk + -> GroupAggregate (cost=223.67..224.14 rows=6 width=48) + Output: (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_item.i_item_sk, tpcds_date_dim.d_date, count(*) + Group Key: (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_item.i_item_sk, tpcds_date_dim.d_date + Filter: (count(*) > 4) + -> Sort (cost=223.67..223.72 rows=18 width=40) + Output: (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_item.i_item_sk, tpcds_date_dim.d_date + Sort Key: (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_item.i_item_sk, tpcds_date_dim.d_date + -> Hash Join (cost=181.20..223.30 rows=18 width=40) + Output: substr((tpcds_item.i_item_desc)::text, 1, 30), tpcds_item.i_item_sk, tpcds_date_dim.d_date + Hash Cond: (tpcds_item.i_item_sk = tpcds_store_sales.ss_item_sk) + -> Seq Scan on public.tpcds_item (cost=0.00..39.33 rows=533 width=422) + Output: tpcds_item.i_item_sk, tpcds_item.i_item_id, tpcds_item.i_rec_start_date, tpcds_item.i_rec_end_date, tpcds_item.i_item_desc, tpcds_item.i_current_price, tpcds_item.i_wholesale_cost, tpcds_item.i_brand_id, tpcds_item.i_brand, tpcds_item.i_class_id, tpcds_item.i_class, tpcds_item.i_category_id, tpcds_item.i_category, tpcds_item.i_manufact_id, tpcds_item.i_manufact, tpcds_item.i_size, tpcds_item.i_formulation, tpcds_item.i_color, tpcds_item.i_units, tpcds_item.i_container, tpcds_item.i_manager_id, tpcds_item.i_product_name + -> Hash (cost=180.98..180.98 rows=18 width=8) + Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=89.63..180.98 rows=18 width=8) + Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date + Hash Key: tpcds_store_sales.ss_item_sk + -> Hash Join (cost=89.63..180.62 rows=18 width=8) + Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date + Inner Unique: true + Hash Cond: (tpcds_store_sales.ss_sold_date_sk = tpcds_date_dim.d_date_sk) + -> Seq Scan on public.tpcds_store_sales (cost=0.00..79.00 rows=4500 width=8) + Output: tpcds_store_sales.ss_sold_date_sk, tpcds_store_sales.ss_sold_time_sk, tpcds_store_sales.ss_item_sk, tpcds_store_sales.ss_customer_sk, tpcds_store_sales.ss_cdemo_sk, tpcds_store_sales.ss_hdemo_sk, tpcds_store_sales.ss_addr_sk, tpcds_store_sales.ss_store_sk, tpcds_store_sales.ss_promo_sk, tpcds_store_sales.ss_ticket_number, tpcds_store_sales.ss_quantity, tpcds_store_sales.ss_wholesale_cost, tpcds_store_sales.ss_list_price, tpcds_store_sales.ss_sales_price, tpcds_store_sales.ss_ext_discount_amt, tpcds_store_sales.ss_ext_sales_price, tpcds_store_sales.ss_ext_wholesale_cost, tpcds_store_sales.ss_ext_list_price, tpcds_store_sales.ss_ext_tax, tpcds_store_sales.ss_coupon_amt, tpcds_store_sales.ss_net_paid, tpcds_store_sales.ss_net_paid_inc_tax, tpcds_store_sales.ss_net_profit + -> Hash (cost=89.08..89.08 rows=44 width=8) + Output: tpcds_date_dim.d_date, tpcds_date_dim.d_date_sk + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..89.08 rows=44 width=8) + Output: tpcds_date_dim.d_date, tpcds_date_dim.d_date_sk + -> Seq Scan on public.tpcds_date_dim (cost=0.00..88.50 rows=15 width=8) + Output: tpcds_date_dim.d_date, tpcds_date_dim.d_date_sk + Filter: (tpcds_date_dim.d_year = ANY ('{1999,2000,2001,2002}'::integer[])) + -> Hash (cost=228.11..228.11 rows=1 width=8) + Output: t1.v1, frequent_ss_items_1.item_sk, (RowIdExpr) + -> Hash Semi Join (cost=227.06..228.11 rows=1 width=8) + Output: t1.v1, frequent_ss_items_1.item_sk, RowIdExpr + Hash Cond: (t1.v1 = frequent_ss_items_1.item_sk) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=4) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=227.05..227.05 rows=1 width=4) + Output: frequent_ss_items_1.item_sk + -> Subquery Scan on frequent_ss_items_1 (cost=226.96..227.04 rows=1 width=4) + Output: frequent_ss_items_1.item_sk + -> GroupAggregate (cost=226.96..227.04 rows=1 width=48) + Output: (substr((tpcds_item_1.i_item_desc)::text, 1, 30)), tpcds_item_1.i_item_sk, tpcds_date_dim_1.d_date, count(*) + Group Key: (substr((tpcds_item_1.i_item_desc)::text, 1, 30)), tpcds_item_1.i_item_sk, tpcds_date_dim_1.d_date + Filter: (count(*) > 4) + -> Sort (cost=226.96..226.97 rows=3 width=40) + Output: (substr((tpcds_item_1.i_item_desc)::text, 1, 30)), tpcds_item_1.i_item_sk, tpcds_date_dim_1.d_date + Sort Key: (substr((tpcds_item_1.i_item_desc)::text, 1, 30)), tpcds_item_1.i_item_sk, tpcds_date_dim_1.d_date + -> Hash Join (cost=132.52..226.94 rows=3 width=40) + Output: substr((tpcds_item_1.i_item_desc)::text, 1, 30), tpcds_item_1.i_item_sk, tpcds_date_dim_1.d_date + Inner Unique: true + Hash Cond: (tpcds_store_sales_1.ss_item_sk = tpcds_item_1.i_item_sk) + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=89.63..183.99 rows=6 width=8) + Output: tpcds_store_sales_1.ss_item_sk, tpcds_date_dim_1.d_date + Hash Key: tpcds_store_sales_1.ss_item_sk + -> Hash Join (cost=89.63..183.87 rows=6 width=8) + Output: tpcds_store_sales_1.ss_item_sk, tpcds_date_dim_1.d_date + Inner Unique: true + Hash Cond: (tpcds_store_sales_1.ss_sold_date_sk = tpcds_date_dim_1.d_date_sk) + -> Seq Scan on public.tpcds_store_sales tpcds_store_sales_1 (cost=0.00..90.25 rows=1500 width=8) + Output: tpcds_store_sales_1.ss_sold_date_sk, tpcds_store_sales_1.ss_sold_time_sk, tpcds_store_sales_1.ss_item_sk, tpcds_store_sales_1.ss_customer_sk, tpcds_store_sales_1.ss_cdemo_sk, tpcds_store_sales_1.ss_hdemo_sk, tpcds_store_sales_1.ss_addr_sk, tpcds_store_sales_1.ss_store_sk, tpcds_store_sales_1.ss_promo_sk, tpcds_store_sales_1.ss_ticket_number, tpcds_store_sales_1.ss_quantity, tpcds_store_sales_1.ss_wholesale_cost, tpcds_store_sales_1.ss_list_price, tpcds_store_sales_1.ss_sales_price, tpcds_store_sales_1.ss_ext_discount_amt, tpcds_store_sales_1.ss_ext_sales_price, tpcds_store_sales_1.ss_ext_wholesale_cost, tpcds_store_sales_1.ss_ext_list_price, tpcds_store_sales_1.ss_ext_tax, tpcds_store_sales_1.ss_coupon_amt, tpcds_store_sales_1.ss_net_paid, tpcds_store_sales_1.ss_net_paid_inc_tax, tpcds_store_sales_1.ss_net_profit + Filter: (tpcds_store_sales_1.ss_item_sk > 0) + -> Hash (cost=89.08..89.08 rows=44 width=8) + Output: tpcds_date_dim_1.d_date, tpcds_date_dim_1.d_date_sk + -> Broadcast Motion 3:3 (slice6; segments: 3) (cost=0.00..89.08 rows=44 width=8) + Output: tpcds_date_dim_1.d_date, tpcds_date_dim_1.d_date_sk + -> Seq Scan on public.tpcds_date_dim tpcds_date_dim_1 (cost=0.00..88.50 rows=15 width=8) + Output: tpcds_date_dim_1.d_date, tpcds_date_dim_1.d_date_sk + Filter: (tpcds_date_dim_1.d_year = ANY ('{1999,2000,2001,2002}'::integer[])) + -> Hash (cost=40.67..40.67 rows=178 width=422) + Output: tpcds_item_1.i_item_desc, tpcds_item_1.i_item_sk + -> Seq Scan on public.tpcds_item tpcds_item_1 (cost=0.00..40.67 rows=178 width=422) + Output: tpcds_item_1.i_item_desc, tpcds_item_1.i_item_sk + Filter: (tpcds_item_1.i_item_sk > 0) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(89 rows) + +-- sql 95 +explain verbose with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from tpcds_web_sales ws1,tpcds_web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2952.11..2952.33 rows=14 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> HashAggregate (cost=2952.11..2952.15 rows=5 width=12) + Output: t1.v1, t1.v2, t1.v3 + Group Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=508.11..2952.10 rows=5 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=508.11..2952.01 rows=5 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + Hash Cond: (ws_wh.ws_order_number = t1.v1) + -> Subquery Scan on ws_wh (cost=179.92..2181.72 rows=35328 width=4) + Output: ws_wh.ws_order_number + -> Hash Join (cost=179.92..2181.72 rows=35328 width=12) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk, ws2.ws_warehouse_sk + Hash Cond: (ws1.ws_order_number = ws2.ws_order_number) + Join Filter: (ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..137.00 rows=3433 width=8) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk + Hash Key: ws1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..68.33 rows=3433 width=8) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk + -> Hash (cost=137.00..137.00 rows=3433 width=8) + Output: ws2.ws_warehouse_sk, ws2.ws_order_number + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..137.00 rows=3433 width=8) + Output: ws2.ws_warehouse_sk, ws2.ws_order_number + Hash Key: ws2.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws2 (cost=0.00..68.33 rows=3433 width=8) + Output: ws2.ws_warehouse_sk, ws2.ws_order_number + -> Hash (cost=328.14..328.14 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number, (RowIdExpr) + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=327.96..328.14 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number, (RowIdExpr) + Hash Key: t1.v1 + -> Result (cost=327.96..328.05 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number, RowIdExpr + -> HashAggregate (cost=327.96..328.00 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number + Group Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice6; segments: 3) (cost=115.19..327.95 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=115.19..327.86 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number, (RowIdExpr) + Hash Cond: (ws_wh_1.ws_order_number = t1.v1) + -> Subquery Scan on ws_wh_1 (cost=114.11..277.62 rows=3925 width=4) + Output: ws_wh_1.ws_order_number + -> Hash Join (cost=114.11..277.62 rows=3925 width=12) + Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk, ws2_1.ws_warehouse_sk + Hash Cond: (ws1_1.ws_order_number = ws2_1.ws_order_number) + Join Filter: (ws1_1.ws_warehouse_sk <> ws2_1.ws_warehouse_sk) + -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..99.81 rows=1144 width=8) + Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk + Hash Key: ws1_1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws1_1 (cost=0.00..76.92 rows=1144 width=8) + Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk + Filter: (ws1_1.ws_order_number > 0) + -> Hash (cost=99.81..99.81 rows=1144 width=8) + Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number + -> Redistribute Motion 3:3 (slice8; segments: 3) (cost=0.00..99.81 rows=1144 width=8) + Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number + Hash Key: ws2_1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws2_1 (cost=0.00..76.92 rows=1144 width=8) + Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number + Filter: (ws2_1.ws_order_number > 0) + -> Hash (cost=1.03..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3, RowIdExpr + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(71 rows) + +explain verbose with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from tpcds_web_sales ws1,tpcds_web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2952.68..2952.91 rows=14 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> HashAggregate (cost=2952.68..2952.73 rows=5 width=12) + Output: t1.v1, t1.v2, t1.v3 + Group Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=508.53..2952.67 rows=5 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=508.53..2952.58 rows=5 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + Hash Cond: (ws_wh.wh1 = t1.v1) + -> Subquery Scan on ws_wh (cost=179.92..2181.72 rows=35328 width=4) + Output: ws_wh.wh1 + -> Hash Join (cost=179.92..2181.72 rows=35328 width=12) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk, ws2.ws_warehouse_sk + Hash Cond: (ws1.ws_order_number = ws2.ws_order_number) + Join Filter: (ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..137.00 rows=3433 width=8) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk + Hash Key: ws1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..68.33 rows=3433 width=8) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk + -> Hash (cost=137.00..137.00 rows=3433 width=8) + Output: ws2.ws_warehouse_sk, ws2.ws_order_number + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..137.00 rows=3433 width=8) + Output: ws2.ws_warehouse_sk, ws2.ws_order_number + Hash Key: ws2.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws2 (cost=0.00..68.33 rows=3433 width=8) + Output: ws2.ws_warehouse_sk, ws2.ws_order_number + -> Hash (cost=328.44..328.44 rows=14 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1, (RowIdExpr) + -> Broadcast Motion 3:3 (slice5; segments: 3) (cost=328.17..328.44 rows=14 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1, (RowIdExpr) + -> Result (cost=328.17..328.26 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1, RowIdExpr + -> HashAggregate (cost=328.17..328.22 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1 + Group Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice6; segments: 3) (cost=115.40..328.16 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=115.40..328.07 rows=5 width=16) + Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1, (RowIdExpr) + Hash Cond: (ws_wh_1.wh1 = t1.v1) + -> Subquery Scan on ws_wh_1 (cost=114.11..277.62 rows=3925 width=4) + Output: ws_wh_1.wh1 + -> Hash Join (cost=114.11..277.62 rows=3925 width=12) + Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk, ws2_1.ws_warehouse_sk + Hash Cond: (ws1_1.ws_order_number = ws2_1.ws_order_number) + Join Filter: (ws1_1.ws_warehouse_sk <> ws2_1.ws_warehouse_sk) + -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..99.81 rows=1144 width=8) + Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk + Hash Key: ws1_1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws1_1 (cost=0.00..76.92 rows=1144 width=8) + Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk + Filter: (ws1_1.ws_order_number > 0) + -> Hash (cost=99.81..99.81 rows=1144 width=8) + Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number + -> Redistribute Motion 3:3 (slice8; segments: 3) (cost=0.00..99.81 rows=1144 width=8) + Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number + Hash Key: ws2_1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws2_1 (cost=0.00..76.92 rows=1144 width=8) + Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number + Filter: (ws2_1.ws_order_number > 0) + -> Hash (cost=1.17..1.17 rows=10 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + -> Broadcast Motion 3:3 (slice9; segments: 3) (cost=0.00..1.17 rows=10 width=12) + Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) + -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + Output: t1.v1, t1.v2, t1.v3, RowIdExpr + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(72 rows) + +-- start_ignore +drop table tpcds_store_sales; +drop table tpcds_date_dim; +drop table tpcds_item; +drop table tpcds_web_sales; +drop table t1; +drop table t2; +-- end_ignore diff --git a/src/test/regress/expected/cte_prune_optimizer.out b/src/test/regress/expected/cte_prune_optimizer.out new file mode 100644 index 00000000000..831f3c9ecc8 --- /dev/null +++ b/src/test/regress/expected/cte_prune_optimizer.out @@ -0,0 +1,1295 @@ +-- start_ignore +drop table if exists t1; +drop table if exists t2; +-- end_ignore +create table t1(v1 int, v2 int, v3 int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t1 values(generate_series(1, 10), generate_series(11, 20), generate_series(21, 30)); +analyze t1; +create table t2(v1 int, v2 int, v3 int); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t2 values(generate_series(0, 100), generate_series(100, 200), generate_series(200, 300)); +-- should pruned both seq scan and shared scan +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=5 width=4) + Output: share0_ref3.v1 + -> Sequence (cost=0.00..1293.00 rows=2 width=4) + Output: share0_ref3.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) + Output: share0_ref1.v1 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=4) + Output: t1.v1 + Filter: (t1.v1 < 5) + -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) + Output: share0_ref3.v1 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Result (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref3.v1 + Filter: (share0_ref3.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref3.v1 + -> Hash (cost=431.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + -> Result (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + Filter: (share0_ref2.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(26 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v1 +---- + 1 + 2 + 3 + 4 +(4 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=5 width=4) + Output: share0_ref3.v2 + -> Sequence (cost=0.00..1293.00 rows=2 width=4) + Output: share0_ref3.v2 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) + Output: share0_ref1.v1, share0_ref1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 5) + -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) + Output: share0_ref3.v2 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Result (cost=0.00..431.00 rows=2 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Filter: (share0_ref3.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + -> Hash (cost=431.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + -> Result (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + Filter: (share0_ref2.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(26 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v2 +---- + 12 + 13 + 14 + 11 +(4 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=5 width=4) + Output: share0_ref3.v3 + -> Sequence (cost=0.00..1293.00 rows=2 width=4) + Output: share0_ref3.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) + Output: share0_ref1.v1, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=8) + Output: t1.v1, t1.v3 + Filter: (t1.v1 < 5) + -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) + Output: share0_ref3.v3 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Result (cost=0.00..431.00 rows=2 width=8) + Output: share0_ref3.v1, share0_ref3.v3 + Filter: (share0_ref3.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=8) + Output: share0_ref3.v1, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + -> Result (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + Filter: (share0_ref2.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(26 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v3 +---- + 22 + 23 + 24 + 21 +(4 rows) + +-- * also should be pruned +explain verbose with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=5 width=4) + Output: share0_ref3.v1 + -> Sequence (cost=0.00..1293.00 rows=2 width=4) + Output: share0_ref3.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) + Output: share0_ref1.v1 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=4) + Output: t1.v1 + Filter: (t1.v1 < 5) + -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) + Output: share0_ref3.v1 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Result (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref3.v1 + Filter: (share0_ref3.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref3.v1 + -> Hash (cost=431.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + -> Result (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + Filter: (share0_ref2.v1 < 5) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=4) + Output: share0_ref2.v1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(26 rows) + +with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + v1 +---- + 1 + 2 + 3 + 4 +(4 rows) + +-- no push filter +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=11 width=4) + Output: share0_ref3.v3 + -> Sequence (cost=0.00..1293.00 rows=4 width=4) + Output: share0_ref3.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) + Output: share0_ref3.v3 + Hash Cond: (share0_ref3.v1 = share0_ref2.v2) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + Hash Key: share0_ref2.v2 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(24 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + v3 +---- + 26 + 25 + 30 + 29 + 21 + 27 + 23 + 28 + 24 + 22 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=11 width=4) + Output: share0_ref3.v2 + -> Sequence (cost=0.00..1293.00 rows=4 width=4) + Output: share0_ref3.v2 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v2 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) + Output: share0_ref3.v2 + Hash Cond: (share0_ref3.v1 = share0_ref2.v2) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + Hash Key: share0_ref2.v2 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(24 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + v2 +---- + 17 + 13 + 18 + 14 + 12 + 16 + 15 + 20 + 19 + 11 +(10 rows) + +-- distribution col can't pruned +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=4) + Output: share0_ref3.v2 + -> Sequence (cost=0.00..1293.00 rows=4 width=4) + Output: share0_ref3.v2 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v2 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) + Output: share0_ref3.v2 + Hash Cond: (share0_ref3.v2 = share0_ref2.v2) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v2 + Hash Key: share0_ref3.v2 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v2 + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v1, share0_ref3.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + Hash Key: share0_ref2.v2 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v2 + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(29 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; + v2 +---- + 18 + 16 + 19 + 13 + 14 + 17 + 11 + 12 + 15 + 20 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=4) + Output: share0_ref3.v3 + -> Sequence (cost=0.00..1293.00 rows=4 width=4) + Output: share0_ref3.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v3 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) + Output: share0_ref3.v3 + Hash Cond: (share0_ref3.v3 = share0_ref2.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v3 + Hash Key: share0_ref3.v3 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v3 + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v1, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v3 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v3 + Hash Key: share0_ref2.v3 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v3 + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(29 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; + v3 +---- + 22 + 24 + 27 + 29 + 28 + 21 + 25 + 23 + 26 + 30 +(10 rows) + +-- groupby/order by/window function/grouping set should be contains in CTE output +-- group by +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=8) + Output: (sum(share0_ref3.v1)) + -> Result (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v1)) + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v1)), share0_ref3.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + Output: t1.v1 + -> GroupAggregate (cost=0.00..862.00 rows=4 width=8) + Output: sum(share0_ref3.v1), share0_ref3.v1 + Group Key: share0_ref3.v1 + -> Sort (cost=0.00..862.00 rows=4 width=4) + Output: share0_ref3.v1 + Sort Key: share0_ref3.v1 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) + Output: share0_ref3.v1 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v1 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(27 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; + sum +----- + 4 + 2 + 8 + 7 + 3 + 10 + 9 + 6 + 5 + 1 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=8) + Output: (sum(share0_ref3.v1)) + -> Result (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v1)) + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v1)), share0_ref3.v2 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v2 + -> GroupAggregate (cost=0.00..862.00 rows=4 width=8) + Output: sum(share0_ref3.v1), share0_ref3.v2 + Group Key: share0_ref3.v2 + -> Sort (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Sort Key: share0_ref3.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Hash Key: share0_ref3.v2 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(30 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + sum +----- + 6 + 9 + 8 + 10 + 2 + 5 + 1 + 3 + 4 + 7 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=8) + Output: (sum(share0_ref3.v3)) + -> Result (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v3)) + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v3)), share0_ref3.v2 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> GroupAggregate (cost=0.00..862.00 rows=4 width=8) + Output: sum(share0_ref3.v3), share0_ref3.v2 + Group Key: share0_ref3.v2 + -> Sort (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Sort Key: share0_ref3.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Hash Key: share0_ref3.v2 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=12) + Output: share0_ref3.v1, share0_ref3.v2, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(30 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + sum +----- + 26 + 29 + 28 + 30 + 22 + 25 + 21 + 23 + 24 + 27 +(10 rows) + +-- order by +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Result (cost=0.00..1293.00 rows=10 width=4) + Output: share0_ref3.v1 + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=4) + Output: share0_ref3.v1, share0_ref2.v1 + Merge Key: share0_ref2.v1 + -> Sort (cost=0.00..1293.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref2.v1 + Sort Key: share0_ref2.v1 + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref2.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + Output: t1.v1 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref2.v1 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v1 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(25 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; + v1 +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; + QUERY PLAN +---------------------------------------------------------------------------------------------------------- + Result (cost=0.00..1293.00 rows=10 width=4) + Output: share0_ref3.v1 + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=4) + Output: share0_ref3.v1, share0_ref2.v3 + Merge Key: share0_ref2.v3 + -> Sort (cost=0.00..1293.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref2.v3 + Sort Key: share0_ref2.v3 + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref2.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v3 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref2.v3 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref3.v1, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=8) + Output: share0_ref2.v1, share0_ref2.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref2.v1, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(25 rows) + +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; + v1 +---- + 1 + 2 + 3 + 4 + 5 + 6 + 7 + 8 + 9 + 10 +(10 rows) + +-- window function +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=8) + Output: (sum(share0_ref3.v1) OVER (?)) + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v1) OVER (?)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v2 + -> Redistribute Motion 1:3 (slice2) (cost=0.00..862.00 rows=4 width=8) + Output: (sum(share0_ref3.v1) OVER (?)) + -> WindowAgg (cost=0.00..862.00 rows=10 width=8) + Output: sum(share0_ref3.v1) OVER (?) + Order By: share0_ref3.v2 + -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..862.00 rows=10 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Merge Key: share0_ref3.v2 + -> Sort (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Sort Key: share0_ref3.v2 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(30 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + sum +----- + 1 + 3 + 6 + 10 + 15 + 21 + 28 + 36 + 45 + 55 +(10 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=8) + Output: (sum(share0_ref3.v2) OVER (?)) + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: (sum(share0_ref3.v2) OVER (?)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Redistribute Motion 1:3 (slice2) (cost=0.00..862.00 rows=4 width=8) + Output: (sum(share0_ref3.v2) OVER (?)) + -> WindowAgg (cost=0.00..862.00 rows=10 width=8) + Output: sum(share0_ref3.v2) OVER (?) + Order By: share0_ref3.v3 + -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..862.00 rows=10 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Merge Key: share0_ref3.v3 + -> Sort (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Sort Key: share0_ref3.v3 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=12) + Output: share0_ref3.v1, share0_ref3.v2, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(30 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + sum +----- + 11 + 23 + 36 + 50 + 65 + 81 + 98 + 116 + 135 + 155 +(10 rows) + +-- grouping set +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v1,c11.v2); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2586.00 rows=21 width=8) + Output: (sum(share1_ref2.v2)) + -> Sequence (cost=0.00..2586.00 rows=7 width=8) + Output: (sum(share1_ref2.v2)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + Output: t1.v1, t1.v2 + -> Sequence (cost=0.00..2155.00 rows=7 width=8) + Output: (sum(share1_ref2.v2)) + -> Shared Scan (share slice:id 1:1) (cost=0.00..862.00 rows=4 width=1) + Output: share1_ref1.v1, share1_ref1.v2 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2 + -> Append (cost=0.00..1293.00 rows=7 width=8) + -> GroupAggregate (cost=0.00..431.00 rows=4 width=8) + Output: sum(share1_ref2.v2) + Group Key: share1_ref2.v1, share1_ref2.v2 + -> Sort (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref2.v1, share1_ref2.v2 + Sort Key: share1_ref2.v1, share1_ref2.v2 + -> Shared Scan (share slice:id 1:1) (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref2.v1, share1_ref2.v2 + -> GroupAggregate (cost=0.00..431.00 rows=4 width=8) + Output: sum(share1_ref3.v2) + Group Key: share1_ref3.v1 + -> Sort (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref3.v1, share1_ref3.v2 + Sort Key: share1_ref3.v1 + -> Shared Scan (share slice:id 1:1) (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref3.v1, share1_ref3.v2 + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: (sum(share1_ref4.v2)) + -> Redistribute Motion 1:3 (slice2) (cost=0.00..431.00 rows=1 width=8) + Output: (sum(share1_ref4.v2)) + -> Finalize Aggregate (cost=0.00..431.00 rows=1 width=8) + Output: sum(share1_ref4.v2) + -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: (PARTIAL sum(share1_ref4.v2)) + -> Partial Aggregate (cost=0.00..431.00 rows=1 width=8) + Output: PARTIAL sum(share1_ref4.v2) + -> Shared Scan (share slice:id 3:1) (cost=0.00..431.00 rows=4 width=4) + Output: share1_ref4.v1, share1_ref4.v2 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(52 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v1,c11.v2); + sum +----- + 11 + 11 + 12 + 12 + 13 + 13 + 14 + 14 + 15 + 15 + 16 + 16 + 17 + 17 + 18 + 18 + 19 + 19 + 20 + 20 + 155 +(21 rows) + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v2,c11.v3); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2586.00 rows=21 width=8) + Output: (sum(share1_ref2.v2)) + -> Sequence (cost=0.00..2586.00 rows=7 width=8) + Output: (sum(share1_ref2.v2)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Sequence (cost=0.00..2155.00 rows=7 width=8) + Output: (sum(share1_ref2.v2)) + -> Shared Scan (share slice:id 1:1) (cost=0.00..862.00 rows=4 width=1) + Output: share1_ref1.v2, share1_ref1.v3 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v2, share0_ref3.v3 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=12) + Output: share0_ref3.v1, share0_ref3.v2, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + -> Append (cost=0.00..1293.00 rows=7 width=8) + -> GroupAggregate (cost=0.00..431.00 rows=4 width=8) + Output: sum(share1_ref2.v2) + Group Key: share1_ref2.v2, share1_ref2.v3 + -> Sort (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref2.v2, share1_ref2.v3 + Sort Key: share1_ref2.v2, share1_ref2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref2.v2, share1_ref2.v3 + Hash Key: share1_ref2.v2, share1_ref2.v3 + -> Shared Scan (share slice:id 2:1) (cost=0.00..431.00 rows=4 width=8) + Output: share1_ref2.v2, share1_ref2.v3 + -> GroupAggregate (cost=0.00..431.00 rows=4 width=8) + Output: sum(share1_ref3.v2) + Group Key: share1_ref3.v2 + -> Sort (cost=0.00..431.00 rows=4 width=4) + Output: share1_ref3.v2 + Sort Key: share1_ref3.v2 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: share1_ref3.v2 + Hash Key: share1_ref3.v2 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share1_ref3.v2 + -> Shared Scan (share slice:id 3:1) (cost=0.00..431.00 rows=4 width=4) + Output: share1_ref3.v2, share1_ref3.v3 + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: (sum(share1_ref4.v2)) + -> Redistribute Motion 1:3 (slice4) (cost=0.00..431.00 rows=1 width=8) + Output: (sum(share1_ref4.v2)) + -> Finalize Aggregate (cost=0.00..431.00 rows=1 width=8) + Output: sum(share1_ref4.v2) + -> Gather Motion 3:1 (slice5; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: (PARTIAL sum(share1_ref4.v2)) + -> Partial Aggregate (cost=0.00..431.00 rows=1 width=8) + Output: PARTIAL sum(share1_ref4.v2) + -> Shared Scan (share slice:id 5:1) (cost=0.00..431.00 rows=4 width=4) + Output: share1_ref4.v2, share1_ref4.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(60 rows) + +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + sum +----- + 11 + 23 + 36 + 50 + 65 + 81 + 98 + 116 + 135 + 155 +(10 rows) + +-- CTE producer should have right output +explain verbose with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1724.00 rows=2 width=4) + Output: share0_ref3.v1 + -> Sequence (cost=0.00..1724.00 rows=1 width=4) + Output: share0_ref3.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..862.00 rows=1 width=1) + Output: share0_ref1.v1, share0_ref1.v1_1 + -> Hash Join (cost=0.00..862.00 rows=1 width=8) + Output: t1.v1, t2.v1 + Hash Cond: (t1.v1 = t2.v1) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + Output: t1.v1 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: t2.v1 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=1 width=4) + Output: t2.v1 + -> Hash Left Join (cost=0.00..862.00 rows=1 width=4) + Output: share0_ref3.v1 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.v1 + Filter: (share0_ref3.v1 = share0_ref3.v1_1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref3.v1, share0_ref3.v1_1 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref2.v1 + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.v1 + Filter: (share0_ref2.v1 = share0_ref2.v1_1) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.v1, share0_ref2.v1_1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(32 rows) + +with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + v1 +---- + 5 + 6 + 9 + 10 + 2 + 3 + 4 + 7 + 8 + 1 +(10 rows) + +explain verbose with c1 as (select sum(v1) as v1, sum(v2) as v2, v3 from t1 group by v3) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + QUERY PLAN +------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=11 width=8) + Output: share0_ref3.v1 + -> Sequence (cost=0.00..1293.00 rows=4 width=8) + Output: share0_ref3.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) + Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 + -> HashAggregate (cost=0.00..431.00 rows=4 width=20) + Output: sum(t1.v1), sum(t1.v2), t1.v3 + Group Key: t1.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + Hash Key: t1.v3 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) + Output: share0_ref3.v1 + Hash Cond: (share0_ref3.v1 = share0_ref2.v1) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1 + Hash Key: share0_ref3.v1 + -> Result (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1 + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref3.v1, share0_ref3.v2, share0_ref3.v3 + -> Hash (cost=431.00..431.00 rows=4 width=8) + Output: share0_ref2.v1 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref2.v1 + Hash Key: share0_ref2.v1 + -> Result (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref2.v1 + -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=4 width=8) + Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(35 rows) + +with c1 as (select lt1.v3 as v3, lt1.v1 as lo1, rt1.v1 as ro1 from t1 lt1, t1 rt1 where lt1.v2 = rt1.v2 and lt1.v1 = rt1.v1) +select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 from c1 where v3 > 0); + v1 | v2 | v3 +----+----+---- +(0 rows) + +-- cte in cte +-- function call +-- TPCDS cte not support reduce producter output yet +-- start_ignore +drop table if exists tpcds_store_sales; +drop table if exists tpcds_date_dim; +drop table if exists tpcds_item; +drop table if exists tpcds_web_sales; +-- end_ignore +create table tpcds_store_sales +( + ss_sold_date_sk integer , + ss_sold_time_sk integer , + ss_item_sk integer not null, + ss_customer_sk integer , + ss_cdemo_sk integer , + ss_hdemo_sk integer , + ss_addr_sk integer , + ss_store_sk integer , + ss_promo_sk integer , + ss_ticket_number integer not null, + ss_quantity integer , + ss_wholesale_cost decimal(7,2) , + ss_list_price decimal(7,2) , + ss_sales_price decimal(7,2) , + ss_ext_discount_amt decimal(7,2) , + ss_ext_sales_price decimal(7,2) , + ss_ext_wholesale_cost decimal(7,2) , + ss_ext_list_price decimal(7,2) , + ss_ext_tax decimal(7,2) , + ss_coupon_amt decimal(7,2) , + ss_net_paid decimal(7,2) , + ss_net_paid_inc_tax decimal(7,2) , + ss_net_profit decimal(7,2) , + primary key (ss_item_sk, ss_ticket_number) +); +create table tpcds_date_dim +( + d_date_sk integer not null, + d_date_id char(16) not null, + d_date date , + d_month_seq integer , + d_week_seq integer , + d_quarter_seq integer , + d_year integer , + d_dow integer , + d_moy integer , + d_dom integer , + d_qoy integer , + d_fy_year integer , + d_fy_quarter_seq integer , + d_fy_week_seq integer , + d_day_name char(9) , + d_quarter_name char(6) , + d_holiday char(1) , + d_weekend char(1) , + d_following_holiday char(1) , + d_first_dom integer , + d_last_dom integer , + d_same_day_ly integer , + d_same_day_lq integer , + d_current_day char(1) , + d_current_week char(1) , + d_current_month char(1) , + d_current_quarter char(1) , + d_current_year char(1) , + primary key (d_date_sk) +); +create table tpcds_item +( + i_item_sk integer not null, + i_item_id char(16) not null, + i_rec_start_date date , + i_rec_end_date date , + i_item_desc varchar(200) , + i_current_price decimal(7,2) , + i_wholesale_cost decimal(7,2) , + i_brand_id integer , + i_brand char(50) , + i_class_id integer , + i_class char(50) , + i_category_id integer , + i_category char(50) , + i_manufact_id integer , + i_manufact char(50) , + i_size char(20) , + i_formulation char(20) , + i_color char(20) , + i_units char(10) , + i_container char(10) , + i_manager_id integer , + i_product_name char(50) , + primary key (i_item_sk) +); +create table tpcds_web_sales +( + ws_sold_date_sk integer , + ws_sold_time_sk integer , + ws_ship_date_sk integer , + ws_item_sk integer not null, + ws_bill_customer_sk integer , + ws_bill_cdemo_sk integer , + ws_bill_hdemo_sk integer , + ws_bill_addr_sk integer , + ws_ship_customer_sk integer , + ws_ship_cdemo_sk integer , + ws_ship_hdemo_sk integer , + ws_ship_addr_sk integer , + ws_web_page_sk integer , + ws_web_site_sk integer , + ws_ship_mode_sk integer , + ws_warehouse_sk integer , + ws_promo_sk integer , + ws_order_number integer not null, + ws_quantity integer , + ws_wholesale_cost decimal(7,2) , + ws_list_price decimal(7,2) , + ws_sales_price decimal(7,2) , + ws_ext_discount_amt decimal(7,2) , + ws_ext_sales_price decimal(7,2) , + ws_ext_wholesale_cost decimal(7,2) , + ws_ext_list_price decimal(7,2) , + ws_ext_tax decimal(7,2) , + ws_coupon_amt decimal(7,2) , + ws_ext_ship_cost decimal(7,2) , + ws_net_paid decimal(7,2) , + ws_net_paid_inc_tax decimal(7,2) , + ws_net_paid_inc_ship decimal(7,2) , + ws_net_paid_inc_ship_tax decimal(7,2) , + ws_net_profit decimal(7,2) , +primary key (ws_item_sk, ws_order_number) +); +-- sql 23 +explain verbose with frequent_ss_items as + (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + from tpcds_store_sales + ,tpcds_date_dim + ,tpcds_item + where ss_sold_date_sk = d_date_sk + and ss_item_sk = i_item_sk + and d_year in (1999,1999+1,1999+2,1999+3) + group by substr(i_item_desc,1,30),i_item_sk,d_date + having count(*) >4) +select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) + and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2161.00 rows=1 width=4) + Output: t1.v1 + -> Sequence (cost=0.00..2161.00 rows=1 width=4) + Output: t1.v1 + -> Shared Scan (share slice:id 1:0) (cost=0.00..868.00 rows=1 width=1) + Output: share0_ref1.itemdesc, share0_ref1.i_item_sk, share0_ref1.d_date, share0_ref1.cnt + -> Result (cost=0.00..868.00 rows=1 width=24) + Output: (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_item.i_item_sk, tpcds_date_dim.d_date, (count()) + Filter: ((count()) > 4) + -> HashAggregate (cost=0.00..868.00 rows=1 width=32) + Output: count(), count(), tpcds_date_dim.d_date, tpcds_item.i_item_sk, (substr((tpcds_item.i_item_desc)::text, 1, 30)) + Group Key: substr((tpcds_item.i_item_desc)::text, 1, 30), tpcds_item.i_item_sk, tpcds_date_dim.d_date + -> Hash Join (cost=0.00..868.00 rows=1 width=16) + Output: substr((tpcds_item.i_item_desc)::text, 1, 30), tpcds_date_dim.d_date, tpcds_item.i_item_sk + Hash Cond: (tpcds_item.i_item_sk = tpcds_store_sales.ss_item_sk) + -> Seq Scan on public.tpcds_item (cost=0.00..431.00 rows=1 width=12) + Output: tpcds_item.i_item_sk, tpcds_item.i_item_desc + -> Hash (cost=437.00..437.00 rows=1 width=8) + Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..437.00 rows=1 width=8) + Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date + Hash Key: tpcds_store_sales.ss_item_sk + -> Nested Loop (cost=0.00..437.00 rows=1 width=8) + Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date + Join Filter: true + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: tpcds_store_sales.ss_sold_date_sk, tpcds_store_sales.ss_item_sk + Hash Key: tpcds_store_sales.ss_sold_date_sk + -> Seq Scan on public.tpcds_store_sales (cost=0.00..431.00 rows=1 width=8) + Output: tpcds_store_sales.ss_sold_date_sk, tpcds_store_sales.ss_item_sk + -> Index Scan using tpcds_date_dim_pkey on public.tpcds_date_dim (cost=0.00..6.00 rows=1 width=4) + Output: tpcds_date_dim.d_date, tpcds_date_dim.d_year + Index Cond: (tpcds_date_dim.d_date_sk = tpcds_store_sales.ss_sold_date_sk) + Filter: (tpcds_date_dim.d_year = ANY ('{1999,2000,2001,2002}'::integer[])) + -> Hash Semi Join (cost=0.00..1293.00 rows=1 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = share0_ref2.i_item_sk) + -> Hash Semi Join (cost=0.00..862.00 rows=1 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = share0_ref3.i_item_sk) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + Output: t1.v1 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref3.i_item_sk + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.i_item_sk + Filter: (share0_ref3.i_item_sk > 0) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.itemdesc, share0_ref3.i_item_sk, share0_ref3.d_date, share0_ref3.cnt + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref2.itemdesc, share0_ref2.i_item_sk, share0_ref2.d_date, share0_ref2.cnt + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.itemdesc, share0_ref2.i_item_sk, share0_ref2.d_date, share0_ref2.cnt + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(55 rows) + +-- sql 95 +explain verbose with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from tpcds_web_sales ws1,tpcds_web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1730.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Sequence (cost=0.00..1730.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..437.00 rows=1 width=1) + Output: share0_ref1.ws_order_number, share0_ref1.ws_warehouse_sk, share0_ref1.ws_warehouse_sk_1 + -> Nested Loop (cost=0.00..437.00 rows=1 width=12) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk, ws2.ws_warehouse_sk + Join Filter: true + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: ws1.ws_warehouse_sk, ws1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..431.00 rows=1 width=8) + Output: ws1.ws_warehouse_sk, ws1.ws_order_number + -> Index Scan using tpcds_web_sales_pkey on public.tpcds_web_sales ws2 (cost=0.00..6.00 rows=1 width=4) + Output: ws2.ws_warehouse_sk + Index Cond: (ws2.ws_order_number = ws1.ws_order_number) + Filter: (ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + -> Hash Semi Join (cost=0.00..1293.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + Hash Cond: (t1.v1 = share0_ref2.ws_order_number) + -> Hash Semi Join (cost=0.00..862.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + Hash Cond: (t1.v1 = share0_ref3.ws_order_number) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_order_number + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_order_number + Hash Key: share0_ref3.ws_order_number + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_order_number + Filter: (share0_ref3.ws_order_number > 0) + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_order_number, share0_ref3.ws_warehouse_sk, share0_ref3.ws_warehouse_sk_1 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_order_number + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_order_number + Hash Key: share0_ref2.ws_order_number + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_order_number + -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_order_number, share0_ref2.ws_warehouse_sk, share0_ref2.ws_warehouse_sk_1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(46 rows) + +explain verbose with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from tpcds_web_sales ws1,tpcds_web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1730.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Sequence (cost=0.00..1730.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Shared Scan (share slice:id 1:0) (cost=0.00..437.00 rows=1 width=1) + Output: share0_ref1.ws_order_number, share0_ref1.ws_warehouse_sk, share0_ref1.ws_warehouse_sk_1 + -> Nested Loop (cost=0.00..437.00 rows=1 width=12) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk, ws2.ws_warehouse_sk + Join Filter: true + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: ws1.ws_warehouse_sk, ws1.ws_order_number + -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..431.00 rows=1 width=8) + Output: ws1.ws_warehouse_sk, ws1.ws_order_number + -> Index Scan using tpcds_web_sales_pkey on public.tpcds_web_sales ws2 (cost=0.00..6.00 rows=1 width=4) + Output: ws2.ws_warehouse_sk + Index Cond: (ws2.ws_order_number = ws1.ws_order_number) + Filter: (ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) + -> Hash Semi Join (cost=0.00..1293.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + Hash Cond: (t1.v1 = share0_ref2.ws_warehouse_sk) + -> Hash Semi Join (cost=0.00..862.00 rows=1 width=12) + Output: t1.v1, t1.v2, t1.v3 + Hash Cond: (t1.v1 = share0_ref3.ws_warehouse_sk) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_warehouse_sk + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_warehouse_sk + Hash Key: share0_ref3.ws_warehouse_sk + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref3.ws_warehouse_sk + Filter: (share0_ref3.ws_order_number > 0) + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref3.ws_order_number, share0_ref3.ws_warehouse_sk, share0_ref3.ws_warehouse_sk_1 + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_warehouse_sk + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_warehouse_sk + Hash Key: share0_ref2.ws_warehouse_sk + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_warehouse_sk + -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=1 width=4) + Output: share0_ref2.ws_order_number, share0_ref2.ws_warehouse_sk, share0_ref2.ws_warehouse_sk_1 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(46 rows) + +-- start_ignore +drop table tpcds_store_sales; +drop table tpcds_date_dim; +drop table tpcds_item; +drop table tpcds_web_sales; +drop table t1; +drop table t2; +-- end_ignore diff --git a/src/test/regress/expected/shared_scan_optimizer.out b/src/test/regress/expected/shared_scan_optimizer.out index a8ab8bd0c23..d782988bbd6 100644 --- a/src/test/regress/expected/shared_scan_optimizer.out +++ b/src/test/regress/expected/shared_scan_optimizer.out @@ -107,12 +107,11 @@ WITH cte AS (SELECT * FROM t1 WHERE random() < 0.1 LIMIT 10) SELECT a, 1, 1 FROM Hash Cond: (share0_ref2.a = t2.a) -> Redistribute Motion 1:3 (slice3) Hash Key: share0_ref2.a - -> Result - -> Shared Scan (share slice:id 3:0) + -> Shared Scan (share slice:id 3:0) -> Hash -> Seq Scan on t2 - Optimizer: Pivotal Optimizer (GPORCA) -(17 rows) + Optimizer: GPORCA +(16 rows) -- This functions returns one more column than expected. CREATE OR REPLACE FUNCTION col_mismatch_func1() RETURNS TABLE (field1 int, field2 int) diff --git a/src/test/regress/sql/cte_prune.sql b/src/test/regress/sql/cte_prune.sql new file mode 100644 index 00000000000..2c16b515a61 --- /dev/null +++ b/src/test/regress/sql/cte_prune.sql @@ -0,0 +1,263 @@ +-- start_ignore +drop table if exists t1; +drop table if exists t2; +-- end_ignore + +create table t1(v1 int, v2 int, v3 int); +insert into t1 values(generate_series(1, 10), generate_series(11, 20), generate_series(21, 30)); +analyze t1; + +create table t2(v1 int, v2 int, v3 int); +insert into t2 values(generate_series(0, 100), generate_series(100, 200), generate_series(200, 300)); + +-- should pruned both seq scan and shared scan +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + +-- * also should be pruned +explain verbose with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; +with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; + +-- no push filter +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; + +-- distribution col can't pruned +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; +with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; +with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; + +-- groupby/order by/window function/grouping set should be contains in CTE output + +-- group by +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; +with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; + +-- order by +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; + +explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; +with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; + +-- window function +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; +with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + +-- grouping set +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v1,c11.v2); +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v1,c11.v2); +explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by ROLLUP(c11.v2,c11.v3); +with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + + +-- CTE producer should have right output + +explain verbose with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + +with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + +explain verbose with c1 as (select sum(v1) as v1, sum(v2) as v2, v3 from t1 group by v3) +select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; + +with c1 as (select lt1.v3 as v3, lt1.v1 as lo1, rt1.v1 as ro1 from t1 lt1, t1 rt1 where lt1.v2 = rt1.v2 and lt1.v1 = rt1.v1) +select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 from c1 where v3 > 0); + +-- cte in cte + + +-- function call + + +-- TPCDS cte not support reduce producter output yet + +-- start_ignore +drop table if exists tpcds_store_sales; +drop table if exists tpcds_date_dim; +drop table if exists tpcds_item; +drop table if exists tpcds_web_sales; +-- end_ignore + +create table tpcds_store_sales +( + ss_sold_date_sk integer , + ss_sold_time_sk integer , + ss_item_sk integer not null, + ss_customer_sk integer , + ss_cdemo_sk integer , + ss_hdemo_sk integer , + ss_addr_sk integer , + ss_store_sk integer , + ss_promo_sk integer , + ss_ticket_number integer not null, + ss_quantity integer , + ss_wholesale_cost decimal(7,2) , + ss_list_price decimal(7,2) , + ss_sales_price decimal(7,2) , + ss_ext_discount_amt decimal(7,2) , + ss_ext_sales_price decimal(7,2) , + ss_ext_wholesale_cost decimal(7,2) , + ss_ext_list_price decimal(7,2) , + ss_ext_tax decimal(7,2) , + ss_coupon_amt decimal(7,2) , + ss_net_paid decimal(7,2) , + ss_net_paid_inc_tax decimal(7,2) , + ss_net_profit decimal(7,2) , + primary key (ss_item_sk, ss_ticket_number) +); + +create table tpcds_date_dim +( + d_date_sk integer not null, + d_date_id char(16) not null, + d_date date , + d_month_seq integer , + d_week_seq integer , + d_quarter_seq integer , + d_year integer , + d_dow integer , + d_moy integer , + d_dom integer , + d_qoy integer , + d_fy_year integer , + d_fy_quarter_seq integer , + d_fy_week_seq integer , + d_day_name char(9) , + d_quarter_name char(6) , + d_holiday char(1) , + d_weekend char(1) , + d_following_holiday char(1) , + d_first_dom integer , + d_last_dom integer , + d_same_day_ly integer , + d_same_day_lq integer , + d_current_day char(1) , + d_current_week char(1) , + d_current_month char(1) , + d_current_quarter char(1) , + d_current_year char(1) , + primary key (d_date_sk) +); + +create table tpcds_item +( + i_item_sk integer not null, + i_item_id char(16) not null, + i_rec_start_date date , + i_rec_end_date date , + i_item_desc varchar(200) , + i_current_price decimal(7,2) , + i_wholesale_cost decimal(7,2) , + i_brand_id integer , + i_brand char(50) , + i_class_id integer , + i_class char(50) , + i_category_id integer , + i_category char(50) , + i_manufact_id integer , + i_manufact char(50) , + i_size char(20) , + i_formulation char(20) , + i_color char(20) , + i_units char(10) , + i_container char(10) , + i_manager_id integer , + i_product_name char(50) , + primary key (i_item_sk) +); + +create table tpcds_web_sales +( + ws_sold_date_sk integer , + ws_sold_time_sk integer , + ws_ship_date_sk integer , + ws_item_sk integer not null, + ws_bill_customer_sk integer , + ws_bill_cdemo_sk integer , + ws_bill_hdemo_sk integer , + ws_bill_addr_sk integer , + ws_ship_customer_sk integer , + ws_ship_cdemo_sk integer , + ws_ship_hdemo_sk integer , + ws_ship_addr_sk integer , + ws_web_page_sk integer , + ws_web_site_sk integer , + ws_ship_mode_sk integer , + ws_warehouse_sk integer , + ws_promo_sk integer , + ws_order_number integer not null, + ws_quantity integer , + ws_wholesale_cost decimal(7,2) , + ws_list_price decimal(7,2) , + ws_sales_price decimal(7,2) , + ws_ext_discount_amt decimal(7,2) , + ws_ext_sales_price decimal(7,2) , + ws_ext_wholesale_cost decimal(7,2) , + ws_ext_list_price decimal(7,2) , + ws_ext_tax decimal(7,2) , + ws_coupon_amt decimal(7,2) , + ws_ext_ship_cost decimal(7,2) , + ws_net_paid decimal(7,2) , + ws_net_paid_inc_tax decimal(7,2) , + ws_net_paid_inc_ship decimal(7,2) , + ws_net_paid_inc_ship_tax decimal(7,2) , + ws_net_profit decimal(7,2) , +primary key (ws_item_sk, ws_order_number) +); + +-- sql 23 +explain verbose with frequent_ss_items as + (select substr(i_item_desc,1,30) itemdesc,i_item_sk item_sk,d_date solddate,count(*) cnt + from tpcds_store_sales + ,tpcds_date_dim + ,tpcds_item + where ss_sold_date_sk = d_date_sk + and ss_item_sk = i_item_sk + and d_year in (1999,1999+1,1999+2,1999+3) + group by substr(i_item_desc,1,30),i_item_sk,d_date + having count(*) >4) +select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) + and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); + +-- sql 95 +explain verbose with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from tpcds_web_sales ws1,tpcds_web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); + +explain verbose with ws_wh as +(select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 + from tpcds_web_sales ws1,tpcds_web_sales ws2 + where ws1.ws_order_number = ws2.ws_order_number + and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) +select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); + +-- start_ignore +drop table tpcds_store_sales; +drop table tpcds_date_dim; +drop table tpcds_item; +drop table tpcds_web_sales; + +drop table t1; +drop table t2; +-- end_ignore From 65cbe8ba0799247b229d211d290af4ae7fd1253a Mon Sep 17 00:00:00 2001 From: zhoujiaqi Date: Wed, 14 May 2025 18:11:44 +0800 Subject: [PATCH 005/244] ORCA: Pruning the output columns in CTE MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit In previous commits, we addressed the issue of "column references not used by producer or consumer." However, in practical cases, if a producer's output column is used as a GROUP BY key, JOIN key, etc., by downstream operators (within CTEs), the corresponding column can not be pruned. This commit adopts a new approach to resolve the issue: 1. We mark columns used in the consumer(Referenced by upper-level operators) and notify the corresponding producer. 2. The producer no longer marks its own output columns as used. Instead, it finalizes its output columns (i.e., the union of all columns required by consumers). 3. Since `NodeSharedScan` does not perform column projection, we align the output columns in consumers (which are guaranteed to be a subset of the producer’s output columns) after the producer finalizes its output. Note: Even if NodeSharedScan supported column projection, it would not improve performance but complicate certain cases. Thus, column projection was intentionally not implemented in `NodeSharedScan`. --- .../regress/expected/gporca_optimizer.out | 5 +- .../expected/shared_scan_optimizer.out | 5 +- .../gpopt/config/CConfigParamMapping.cpp | 6 +- .../gpopt/translate/CContextDXLToPlStmt.cpp | 45 +- .../translate/CTranslatorDXLToPlStmt.cpp | 41 +- .../gpopt/translate/CTranslatorQueryToDXL.cpp | 2 +- .../libgpopt/include/gpopt/base/CCTEInfo.h | 29 +- .../gpopt/operators/CLogicalCTEConsumer.h | 14 + .../gpopt/operators/CLogicalCTEProducer.h | 35 +- .../gpopt/operators/CPhysicalCTEConsumer.h | 11 + .../gpopt/operators/CPhysicalCTEProducer.h | 11 +- .../gpopt/translate/CTranslatorDXLToExpr.h | 9 +- .../gporca/libgpopt/src/base/CCTEInfo.cpp | 72 +- .../src/operators/CExpressionPreprocessor.cpp | 105 ++- .../src/operators/CLogicalCTEConsumer.cpp | 79 +- .../src/operators/CLogicalCTEProducer.cpp | 35 +- .../src/operators/COrderedAggPreprocessor.cpp | 2 + .../src/operators/CPhysicalCTEConsumer.cpp | 15 +- .../src/operators/CPhysicalCTEProducer.cpp | 59 +- .../src/translate/CTranslatorDXLToExpr.cpp | 197 +++-- .../src/translate/CTranslatorExprToDXL.cpp | 13 +- .../src/xforms/CXformExpandFullOuterJoin.cpp | 7 +- .../src/xforms/CXformGbAggWithMDQA2Join.cpp | 1 + .../src/xforms/CXformImplementCTEProducer.cpp | 2 +- .../src/xforms/CXformInlineCTEConsumer.cpp | 1 + .../CXformInlineCTEConsumerUnderSelect.cpp | 1 + .../libgpopt/src/xforms/CXformUnnestTVF.cpp | 1 + .../libgpopt/src/xforms/CXformUtils.cpp | 8 +- .../dxl/operators/CDXLPhysicalCTEConsumer.h | 12 +- .../dxl/operators/CDXLPhysicalCTEProducer.h | 13 +- .../include/naucrates/dxl/xml/dxltokens.h | 1 + .../include/naucrates/traceflags/traceflags.h | 5 +- .../src/operators/CDXLPhysicalCTEConsumer.cpp | 14 +- .../src/operators/CDXLPhysicalCTEProducer.cpp | 14 +- .../CParseHandlerPhysicalCTEConsumer.cpp | 2 +- .../CParseHandlerPhysicalCTEProducer.cpp | 7 +- .../gporca/libnaucrates/src/xml/dxltokens.cpp | 1 + .../src/unittest/gpopt/xforms/CXformTest.cpp | 1 + src/backend/utils/misc/guc_gp.c | 14 +- .../gpopt/translate/CContextDXLToPlStmt.h | 47 +- src/include/utils/guc.h | 1 + src/include/utils/unsync_guc_name.h | 1 + src/test/regress/expected/cte_prune.out | 775 +++++++++++++--- .../regress/expected/cte_prune_optimizer.out | 829 +++++++++++++++--- .../regress/expected/gp_dqa_optimizer.out | 10 +- .../regress/expected/gporca_optimizer.out | 23 +- .../expected/shared_scan_optimizer.out | 5 +- src/test/regress/parallel_schedule | 2 +- src/test/regress/sql/cte_prune.sql | 236 ++++- 49 files changed, 2317 insertions(+), 507 deletions(-) diff --git a/contrib/pax_storage/src/test/regress/expected/gporca_optimizer.out b/contrib/pax_storage/src/test/regress/expected/gporca_optimizer.out index d638a97071b..1bc1e8f4040 100644 --- a/contrib/pax_storage/src/test/regress/expected/gporca_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/gporca_optimizer.out @@ -11280,10 +11280,9 @@ EXPLAIN (COSTS OFF) WITH abc AS (SELECT onetimefilter1.a, onetimefilter1.b FROM Filter: (share0_ref3.b = f1.b) -> Materialize -> Broadcast Motion 3:3 (slice6; segments: 3) - -> Result - -> Shared Scan (share slice:id 6:0) + -> Shared Scan (share slice:id 6:0) Optimizer: GPORCA -(43 rows) +(42 rows) WITH abc AS (SELECT onetimefilter1.a, onetimefilter1.b FROM onetimefilter1, onetimefilter2 WHERE onetimefilter1.a=onetimefilter2.a) SELECT (SELECT 1 FROM abc WHERE f1.b = f2.b LIMIT 1), COALESCE((SELECT 2 FROM abc WHERE f1.a=random() AND f1.a=2), 0), (SELECT b FROM abc WHERE b=f1.b) FROM onetimefilter1 f1, onetimefilter2 f2 WHERE f1.b = f2.b; ?column? | coalesce | b diff --git a/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out b/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out index d782988bbd6..a256c71063a 100644 --- a/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out +++ b/contrib/pax_storage/src/test/regress/expected/shared_scan_optimizer.out @@ -107,11 +107,12 @@ WITH cte AS (SELECT * FROM t1 WHERE random() < 0.1 LIMIT 10) SELECT a, 1, 1 FROM Hash Cond: (share0_ref2.a = t2.a) -> Redistribute Motion 1:3 (slice3) Hash Key: share0_ref2.a - -> Shared Scan (share slice:id 3:0) + -> Result + -> Shared Scan (share slice:id 3:0) -> Hash -> Seq Scan on t2 Optimizer: GPORCA -(16 rows) +(17 rows) -- This functions returns one more column than expected. CREATE OR REPLACE FUNCTION col_mismatch_func1() RETURNS TABLE (field1 int, field2 int) diff --git a/src/backend/gpopt/config/CConfigParamMapping.cpp b/src/backend/gpopt/config/CConfigParamMapping.cpp index 2ca64bb959f..cd13d4c6ac4 100644 --- a/src/backend/gpopt/config/CConfigParamMapping.cpp +++ b/src/backend/gpopt/config/CConfigParamMapping.cpp @@ -81,9 +81,13 @@ CConfigParamMapping::SConfigMappingElem CConfigParamMapping::m_elements[] = { false, // m_negate_param GPOS_WSZ_LIT("Prints optimization stats.")}, + {EopttracePrintPreProcessResult, &optimizer_print_preprocess_result, + false, // m_negate_param + GPOS_WSZ_LIT("Prints the expression tree produced by the optimizer preprocess(every steps). Only worked with debug version of CBDB.")}, + {EopttraceDebugCTE, &optimizer_debug_cte, false, // m_negate_param - GPOS_WSZ_LIT("Print debug info of CTE.")}, + GPOS_WSZ_LIT("Print debug info of CTE. Only worked with debug version of CBDB.")}, {EopttraceMinidump, // GPDB_91_MERGE_FIXME: I turned optimizer_minidump from bool into diff --git a/src/backend/gpopt/translate/CContextDXLToPlStmt.cpp b/src/backend/gpopt/translate/CContextDXLToPlStmt.cpp index b0d7caef331..376810a27f1 100644 --- a/src/backend/gpopt/translate/CContextDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CContextDXLToPlStmt.cpp @@ -58,7 +58,7 @@ CContextDXLToPlStmt::CContextDXLToPlStmt( m_distribution_policy(nullptr), m_part_selector_to_param_map(nullptr) { - m_cte_consumer_info = GPOS_NEW(m_mp) HMUlCTEConsumerInfo(m_mp); + m_cte_producer_info = GPOS_NEW(m_mp) HMUlCTEProducerInfo(m_mp); m_part_selector_to_param_map = GPOS_NEW(m_mp) UlongToUlongMap(m_mp); m_used_rte_indexes = GPOS_NEW(m_mp) HMUlIndex(m_mp); } @@ -73,7 +73,7 @@ CContextDXLToPlStmt::CContextDXLToPlStmt( //--------------------------------------------------------------------------- CContextDXLToPlStmt::~CContextDXLToPlStmt() { - m_cte_consumer_info->Release(); + m_cte_producer_info->Release(); m_part_selector_to_param_map->Release(); m_used_rte_indexes->Release(); } @@ -152,52 +152,37 @@ CContextDXLToPlStmt::GetParamTypes() //--------------------------------------------------------------------------- // @function: -// CContextDXLToPlStmt::AddCTEConsumerInfo +// CContextDXLToPlStmt::RegisterCTEProducerPlan // // @doc: -// Add information about the newly found CTE entry +// Register information about the CTE producer // //--------------------------------------------------------------------------- void -CContextDXLToPlStmt::AddCTEConsumerInfo(ULONG cte_id, - ShareInputScan *share_input_scan) +CContextDXLToPlStmt::RegisterCTEProducerInfo(ULONG cte_id, + ULongPtrArray *producer_output_colidx_map, ShareInputScan *siscan) { - GPOS_ASSERT(nullptr != share_input_scan); - - SCTEConsumerInfo *cte_info = m_cte_consumer_info->Find(&cte_id); - if (nullptr != cte_info) - { - cte_info->AddCTEPlan(share_input_scan); - return; - } - - List *cte_plan = ListMake1(share_input_scan); - ULONG *key = GPOS_NEW(m_mp) ULONG(cte_id); - BOOL result GPOS_ASSERTS_ONLY = m_cte_consumer_info->Insert( - key, GPOS_NEW(m_mp) SCTEConsumerInfo(cte_plan)); + BOOL result GPOS_ASSERTS_ONLY = m_cte_producer_info->Insert( + key, GPOS_NEW(m_mp) SCTEEntryInfo(producer_output_colidx_map, siscan)); GPOS_ASSERT(result); } //--------------------------------------------------------------------------- // @function: -// CContextDXLToPlStmt::GetCTEConsumerList +// CContextDXLToPlStmt::GetCTEProducerInfo // // @doc: -// Return the list of GPDB plan nodes representing the CTE consumers -// with the given CTE identifier +// Return the producer in CTE //--------------------------------------------------------------------------- -List * -CContextDXLToPlStmt::GetCTEConsumerList(ULONG cte_id) const +std::pair +CContextDXLToPlStmt::GetCTEProducerInfo(ULONG cte_id) const { - SCTEConsumerInfo *cte_info = m_cte_consumer_info->Find(&cte_id); - if (nullptr != cte_info) - { - return cte_info->m_cte_consumer_list; - } + SCTEEntryInfo *sctepinfo = m_cte_producer_info->Find(&cte_id); + GPOS_ASSERT(sctepinfo); - return nullptr; + return std::make_pair(sctepinfo->m_pidxmap, sctepinfo->m_cte_producer_plan); } //--------------------------------------------------------------------------- diff --git a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp index 3c21b1176b6..3e94e8d77f0 100644 --- a/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp +++ b/src/backend/gpopt/translate/CTranslatorDXLToPlStmt.cpp @@ -4076,11 +4076,12 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEProducerToSharedScan( ShareInputScan *shared_input_scan = MakeNode(ShareInputScan); shared_input_scan->share_id = cte_id; shared_input_scan->discard_output = true; + Plan *plan = &(shared_input_scan->scan.plan); plan->plan_node_id = m_dxl_to_plstmt_context->GetNextPlanId(); - // store share scan node for the translation of CTE Consumers - m_dxl_to_plstmt_context->AddCTEConsumerInfo(cte_id, shared_input_scan); + m_dxl_to_plstmt_context->RegisterCTEProducerInfo(cte_id, + cte_prod_dxlop->GetOutputColIdxMap(), shared_input_scan); // translate cost of the producer TranslatePlanCosts(cte_producer_dxlnode, plan); @@ -4130,7 +4131,16 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEConsumerToSharedScan( CDXLPhysicalCTEConsumer *cte_consumer_dxlop = CDXLPhysicalCTEConsumer::Cast(cte_consumer_dxlnode->GetOperator()); ULONG cte_id = cte_consumer_dxlop->Id(); + ULongPtrArray *output_colidx_map = cte_consumer_dxlop->GetOutputColIdxMap(); + + // get the producer idx map + ULongPtrArray *producer_colidx_map; + ShareInputScan *share_input_scan_cte_producer; + + std::tie(producer_colidx_map, share_input_scan_cte_producer) + = m_dxl_to_plstmt_context->GetCTEProducerInfo(cte_id); + // init the consumer plan ShareInputScan *share_input_scan_cte_consumer = MakeNode(ShareInputScan); share_input_scan_cte_consumer->share_id = cte_id; share_input_scan_cte_consumer->discard_output = false; @@ -4153,6 +4163,17 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEConsumerToSharedScan( GPOS_ASSERT(num_of_proj_list_elem == output_colids_array->Size()); for (ULONG ul = 0; ul < num_of_proj_list_elem; ul++) { + AttrNumber varattno = (AttrNumber)ul + 1; + if (output_colidx_map) { + ULONG remapping_idx; + remapping_idx = *(*output_colidx_map)[ul]; + if (producer_colidx_map) { + remapping_idx = *(*producer_colidx_map)[remapping_idx]; + } + GPOS_ASSERT(remapping_idx != gpos::ulong_max); + varattno = (AttrNumber)remapping_idx + 1; + } + CDXLNode *proj_elem_dxlnode = (*project_list_dxlnode)[ul]; CDXLScalarProjElem *sc_proj_elem_dxlop = CDXLScalarProjElem::Cast(proj_elem_dxlnode->GetOperator()); @@ -4165,7 +4186,7 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEConsumerToSharedScan( OID oid_type = CMDIdGPDB::CastMdid(sc_ident_dxlop->MdidType())->Oid(); Var *var = - gpdb::MakeVar(OUTER_VAR, (AttrNumber)(ul + 1), oid_type, + gpdb::MakeVar(OUTER_VAR, varattno, oid_type, sc_ident_dxlop->TypeModifier(), 0 /* varlevelsup */); CHAR *resname = CTranslatorUtils::CreateMultiByteCharStringFromWCString( @@ -4181,9 +4202,17 @@ CTranslatorDXLToPlStmt::TranslateDXLCTEConsumerToSharedScan( SetParamIds(plan); - // store share scan node for the translation of CTE Consumers - m_dxl_to_plstmt_context->AddCTEConsumerInfo(cte_id, - share_input_scan_cte_consumer); + // DON'T REMOVE, if current consumer need projection, then we can direct add it. + // we still keep the path of projection in consumer + + // Plan *producer_plan = &(share_input_scan_cte_producer->scan.plan); + // if (output_colidx_map != nullptr) { + // share_input_scan_cte_consumer->need_projection = true; + // share_input_scan_cte_consumer->producer_targetlist = gpdb::ListCopy(producer_plan->targetlist); + // if (!share_input_scan_cte_consumer->producer_targetlist) { + // share_input_scan_cte_consumer->need_projection = false; + // } + // } return (Plan *) share_input_scan_cte_consumer; } diff --git a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp index 808277f5e0e..1b7902f5b3b 100644 --- a/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp +++ b/src/backend/gpopt/translate/CTranslatorQueryToDXL.cpp @@ -4787,7 +4787,7 @@ CTranslatorQueryToDXL::ConstructCTEProducerList(List *cte_list, CDXLLogicalCTEProducer *lg_cte_prod_dxlop = GPOS_NEW(m_mp) CDXLLogicalCTEProducer( m_mp, m_context->m_cte_id_counter->next_id(), colid_array, - true /*could_be_pruned*/); + !GPOS_FTRACE(EopttraceEnableCTEInlining) /*can_be_pruned*/); CDXLNode *cte_producer_dxlnode = GPOS_NEW(m_mp) CDXLNode(m_mp, lg_cte_prod_dxlop, cte_child_dxlnode); diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h b/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h index 831ee221373..bf72a444734 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CCTEInfo.h @@ -121,6 +121,9 @@ class CCTEInfo : public CRefCount // logical producer expression CExpression *m_pexprCTEProducer; + // logical consumer expression array + CExpressionArray *m_pexprCTEConsumer; + // map columns of all created consumers of current CTE to their positions in consumer output ColRefToUlongMap *m_phmcrulConsumers; @@ -139,13 +142,20 @@ class CCTEInfo : public CRefCount // dtor ~CCTEInfoEntry() override; - // CTE expression + // CTE producer expression CExpression * - Pexpr() const + PexprProducer() const { return m_pexprCTEProducer; } + // CTE consumer expressions array + CExpressionArray * + PexprsConsumer() const + { + return m_pexprCTEConsumer; + } + // is this CTE used? BOOL FUsed() const @@ -170,6 +180,9 @@ class CCTEInfo : public CRefCount m_fUsed = true; } + // add consumer into the entry + void AddConsumer(CExpression * consumer); + // add given columns to consumers column map void AddConsumerCols(CColRefArray *colref_array); @@ -254,6 +267,9 @@ class CCTEInfo : public CRefCount // number of CTE consumers of given CTE ULONG UlConsumers(ULONG ulCTEId) const; + // logical cte consumer with given id + CExpressionArray *PexprCTEConsumer(ULONG ulCTEId) const; + // check if given CTE is used BOOL FUsed(ULONG ulCTEId) const; @@ -269,6 +285,9 @@ class CCTEInfo : public CRefCount // add cte producer to hashmap void AddCTEProducer(CExpression *pexprCTEProducer); + + // add cte consumer to hashmap + void AddCTEConsumer(CExpression *pexprCTEProducer); // replace cte producer with given expression void ReplaceCTEProducer(CExpression *pexprCTEProducer); @@ -298,6 +317,11 @@ class CCTEInfo : public CRefCount return m_ulNextCTEId++; } + ULONG + CTESize() { + return m_ulNextCTEId; + } + // derive the statistics on the CTE producer void DeriveProducerStats( CLogicalCTEConsumer *popConsumer, // CTE Consumer operator @@ -327,6 +351,7 @@ class CCTEInfo : public CRefCount // mark unused CTEs void MarkUnusedCTEs(); + // walk the producer expressions and add the mapping between computed column // and their corresponding used column(s) void MapComputedToUsedCols(CColumnFactory *col_factory) const; diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEConsumer.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEConsumer.h index 650d19544a9..80365da222b 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEConsumer.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEConsumer.h @@ -47,6 +47,13 @@ class CLogicalCTEConsumer : public CLogical // create the inlined version of this consumer as well as the column mapping void CreateInlinedExpr(CMemoryPool *mp); + // is pruned + BOOL m_pruned; + + void MarkAsNotPruned() { + m_pruned = false; + } + public: CLogicalCTEConsumer(const CLogicalCTEConsumer &) = delete; @@ -59,6 +66,12 @@ class CLogicalCTEConsumer : public CLogical // dtor ~CLogicalCTEConsumer() override; + void MarkAsPruned() { + m_pruned = true; + } + + void ApplyInline(); + // ident accessors EOperatorId Eopid() const override @@ -93,6 +106,7 @@ class CLogicalCTEConsumer : public CLogical return m_phmulcr; } + // get the inlined expression CExpression * PexprInlined() const { diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEProducer.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEProducer.h index 812db8c9db7..7d25d8d7251 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEProducer.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CLogicalCTEProducer.h @@ -38,6 +38,12 @@ class CLogicalCTEProducer : public CLogical // output columns, same as cte columns but in CColRefSet CColRefSet *m_pcrsOutput; + // can be pruned + BOOL m_cbp; + + // used mask + BOOL *m_umask; + public: CLogicalCTEProducer(const CLogicalCTEProducer &) = delete; @@ -45,7 +51,8 @@ class CLogicalCTEProducer : public CLogical explicit CLogicalCTEProducer(CMemoryPool *mp); // ctor - CLogicalCTEProducer(CMemoryPool *mp, ULONG id, CColRefArray *colref_array); + CLogicalCTEProducer(CMemoryPool *mp, ULONG id, CColRefArray *colref_array, + BOOL canbepruned = false); // dtor ~CLogicalCTEProducer() override; @@ -70,6 +77,29 @@ class CLogicalCTEProducer : public CLogical return m_id; } + void + MarkALLUsed() { + ULONG colref_size = m_pdrgpcr->Size(); + for (ULONG index = 0; index < colref_size; index++) { + CColRef *col_ref = (*m_pdrgpcr)[index]; + col_ref->MarkAsUsed(); + } + m_cbp = false; + } + + + BOOL + CanBePruned() const + { + return m_cbp; + } + + BOOL * + UsedMask() const + { + return m_umask; + } + // cte columns CColRefArray * Pdrgpcr() const @@ -84,6 +114,9 @@ class CLogicalCTEProducer : public CLogical return m_pcrsOutput; } + // Recalculate the output columns + void RecalOutputColumns(BOOL *umask, ULONG sz); + // operator specific hash function ULONG HashValue() const override; diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEConsumer.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEConsumer.h index 0d39de496fe..0a618b51f38 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEConsumer.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEConsumer.h @@ -37,6 +37,11 @@ class CPhysicalCTEConsumer : public CPhysical // hashmap for all the columns in the CTE expression UlongToColRefMap *m_phmulcr; + // mapping index of colref to origin idx + // if current colref not used, then set -1 in the position + // if all used, then m_pidxmap will be nullptr + ULongPtrArray *m_pidxmap; + public: CPhysicalCTEConsumer(const CPhysicalCTEConsumer &) = delete; @@ -81,6 +86,12 @@ class CPhysicalCTEConsumer : public CPhysical return m_phmulcr; } + ULongPtrArray * + PCTEIdxMap() const + { + return m_pidxmap; + } + // operator specific hash function ULONG HashValue() const override; diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h index d9ed18b871c..e611e9403e8 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CPhysicalCTEProducer.h @@ -40,11 +40,14 @@ class CPhysicalCTEProducer : public CPhysical // used to record unused columns in m_pdrgpcr ULongPtrArray *m_pdrgpcr_unused; + // used to mapping col index + ULongPtrArray *m_pidxmap; + public: CPhysicalCTEProducer(const CPhysicalCTEProducer &) = delete; // ctor - CPhysicalCTEProducer(CMemoryPool *mp, ULONG id, CColRefArray *colref_array); + CPhysicalCTEProducer(CMemoryPool *mp, ULONG id, CColRefArray *colref_array, BOOL *umask); // dtor ~CPhysicalCTEProducer() override; @@ -84,6 +87,12 @@ class CPhysicalCTEProducer : public CPhysical return m_pdrgpcr_unused; } + ULongPtrArray * + PCTEIdxMap() const + { + return m_pidxmap; + } + // operator specific hash function ULONG HashValue() const override; diff --git a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h index 47147f5bd39..31b0b5f1e22 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h +++ b/src/backend/gporca/libgpopt/include/gpopt/translate/CTranslatorDXLToExpr.h @@ -347,6 +347,9 @@ class CTranslatorDXLToExpr CExpression *Pexpr(const CDXLNode *dxlnode, const CDXLNodeArray *query_output_dxlnode_array, const CDXLNodeArray *cte_producers); + + // Prune the CTEs producer and consumer after DXL tree translated + void PruneCTEs(); // translate children of a DXL node CExpressionArray *PdrgpexprChildren(const CDXLNode *dxlnode); @@ -380,11 +383,7 @@ class CTranslatorDXLToExpr // after the colref in CTE consumer marked used, the producer should marked // the relatived colref as used. - void MarkCTEConsumerColAsUsed(UlongToColRefMap *cidcrCTE, UlongToColRefArrayMap *pidcrsCTE, ULONG colid); - - // after the colref in CTE producer marked used, the consumer should marked - // the relatived colref as used. - void MarkCTEProducerColAsUsed(UlongToColRefMap *cidcrCTE, UlongToColRefArrayMap *pidcrsCTE, ULONG colid); + void MarkCTEConsumerColAsUsed(UlongToColRefMap *mcidcrCTE, ULONG colid); public: // ctor diff --git a/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp b/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp index 335da5049fa..b6423c0073f 100644 --- a/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp +++ b/src/backend/gporca/libgpopt/src/base/CCTEInfo.cpp @@ -35,6 +35,7 @@ CCTEInfo::CCTEInfoEntry::CCTEInfoEntry(CMemoryPool *mp, CExpression *pexprCTEProducer) : m_mp(mp), m_pexprCTEProducer(pexprCTEProducer), + m_pexprCTEConsumer(nullptr), m_phmcrulConsumers(nullptr), m_fUsed(true), m_hasOuterReferences(false) @@ -42,6 +43,7 @@ CCTEInfo::CCTEInfoEntry::CCTEInfoEntry(CMemoryPool *mp, GPOS_ASSERT(nullptr != mp); GPOS_ASSERT(nullptr != pexprCTEProducer); + m_pexprCTEConsumer = GPOS_NEW(mp) CExpressionArray(mp); m_phmcrulConsumers = GPOS_NEW(mp) ColRefToUlongMap(mp); } @@ -59,6 +61,7 @@ CCTEInfo::CCTEInfoEntry::CCTEInfoEntry(CMemoryPool *mp, BOOL fUsed) : m_mp(mp), m_pexprCTEProducer(pexprCTEProducer), + m_pexprCTEConsumer(nullptr), m_phmcrulConsumers(nullptr), m_fUsed(fUsed), m_hasOuterReferences(false) @@ -66,6 +69,7 @@ CCTEInfo::CCTEInfoEntry::CCTEInfoEntry(CMemoryPool *mp, GPOS_ASSERT(nullptr != mp); GPOS_ASSERT(nullptr != pexprCTEProducer); + m_pexprCTEConsumer = GPOS_NEW(mp) CExpressionArray(mp); m_phmcrulConsumers = GPOS_NEW(mp) ColRefToUlongMap(mp); } @@ -81,10 +85,26 @@ CCTEInfo::CCTEInfoEntry::CCTEInfoEntry(CMemoryPool *mp, CCTEInfo::CCTEInfoEntry::~CCTEInfoEntry() { m_pexprCTEProducer->Release(); + m_pexprCTEConsumer->Release(); m_phmcrulConsumers->Release(); } +//--------------------------------------------------------------------------- +// @function: +// CCTEInfo::CCTEInfoEntry::AddConsumerCols +// +// @doc: +// Add given consumer to the consumers array +// +//--------------------------------------------------------------------------- +void CCTEInfo::CCTEInfoEntry::AddConsumer(CExpression * consumer) +{ + GPOS_ASSERT(consumer->Pop() && consumer->Pop()->Eopid() + == COperator::EopLogicalCTEConsumer); + m_pexprCTEConsumer->Append(consumer); +} + //--------------------------------------------------------------------------- // @function: // CCTEInfo::CCTEInfoEntry::AddConsumerCols @@ -253,6 +273,27 @@ CCTEInfo::AddCTEProducer(CExpression *pexprCTEProducer) GPOS_ASSERT(fInserted); } +//--------------------------------------------------------------------------- +// @function: +// CCTEInfo::AddCTEProducer +// +// @doc: +// Add CTE producer to hashmap +// +//--------------------------------------------------------------------------- +void +CCTEInfo::AddCTEConsumer(CExpression *pexprCTEConsumer) +{ + COperator *pop = pexprCTEConsumer->Pop(); + ULONG ulCTEId = CLogicalCTEConsumer::PopConvert(pop)->UlCTEId(); + + CCTEInfoEntry *infoentry = m_phmulcteinfoentry->Find(&ulCTEId); + // must exist + GPOS_ASSERT(infoentry); + pexprCTEConsumer->AddRef(); + infoentry->AddConsumer(pexprCTEConsumer); +} + //--------------------------------------------------------------------------- // @function: @@ -272,7 +313,7 @@ CCTEInfo::ReplaceCTEProducer(CExpression *pexprCTEProducer) GPOS_ASSERT(nullptr != pcteinfoentry); #ifdef GPOS_DBUG - CExpression *pexprCTEProducerOld = pcteinfoentry->Pexpr(); + CExpression *pexprCTEProducerOld = pcteinfoentry->PexprProducer(); COperator *popCTEProducerOld = pexprCTEProducerOld->Pop(); GPOS_ASSERT(ulCTEId == CLogicalCTEProducer::PopConvert(popCTEProducerOld)->UlCTEId()); @@ -328,7 +369,7 @@ CCTEInfo::DeriveProducerStats(CLogicalCTEConsumer *popConsumer, CCTEInfoEntry *pcteinfoentry = m_phmulcteinfoentry->Find(&ulCTEId); GPOS_ASSERT(nullptr != pcteinfoentry); - CExpression *pexprCTEProducer = pcteinfoentry->Pexpr(); + CExpression *pexprCTEProducer = pcteinfoentry->PexprProducer(); // Given the subset of CTE consumer columns needed for statistics derivation, // compute its corresponding set of columns in the CTE Producer @@ -358,7 +399,17 @@ CCTEInfo::PexprCTEProducer(ULONG ulCTEId) const const CCTEInfoEntry *pcteinfoentry = m_phmulcteinfoentry->Find(&ulCTEId); GPOS_ASSERT(nullptr != pcteinfoentry); - return pcteinfoentry->Pexpr(); + return pcteinfoentry->PexprProducer(); +} + +// logical cte consumer with given id +CExpressionArray * +CCTEInfo::PexprCTEConsumer(ULONG ulCTEId) const +{ + const CCTEInfoEntry *pcteinfoentry = m_phmulcteinfoentry->Find(&ulCTEId); + GPOS_ASSERT(nullptr != pcteinfoentry); + + return pcteinfoentry->PexprsConsumer(); } @@ -551,7 +602,7 @@ CCTEInfo::PdrgPexpr(CMemoryPool *mp) const UlongToCTEInfoEntryMapIter hmulei(m_phmulcteinfoentry); while (hmulei.Advance()) { - CExpression *pexpr = hmulei.Value()->Pexpr(); + CExpression *pexpr = hmulei.Value()->PexprProducer(); pexpr->AddRef(); pdrgpexpr->Append(pexpr); } @@ -575,7 +626,7 @@ CCTEInfo::MapComputedToUsedCols(CColumnFactory *col_factory) const UlongToCTEInfoEntryMapIter hmulei(m_phmulcteinfoentry); while (hmulei.Advance()) { - CExpression *pexprProducer = hmulei.Value()->Pexpr(); + CExpression *pexprProducer = hmulei.Value()->PexprProducer(); GPOS_ASSERT(nullptr != pexprProducer); CQueryContext::MapComputedToUsedCols(col_factory, pexprProducer); } @@ -736,16 +787,19 @@ CCTEInfo::PhmulcrConsumerToProducer( { CColRef *colref = crsi.Pcr(); // pruned colref no need do the mapping - if (colref->GetUsage() != CColRef::EUsed) { + if (colref->GetUsage(true, true) != CColRef::EUsed) { continue; } ULONG ulPos = UlConsumerColPos(ulCTEId, colref); if (gpos::ulong_max != ulPos) { - GPOS_ASSERT(ulPos < pdrgpcrUnusedProducer->Size()); - ULONG remapUlPos = ulPos - *(*pdrgpcrUnusedProducer)[ulPos]; - GPOS_ASSERT(remapUlPos < pdrgpcrProducer->Size()); + ULONG remapUlPos = ulPos; + if (pdrgpcrUnusedProducer) { + GPOS_ASSERT(ulPos < pdrgpcrUnusedProducer->Size()); + remapUlPos = ulPos - *(*pdrgpcrUnusedProducer)[ulPos]; + GPOS_ASSERT(remapUlPos < pdrgpcrProducer->Size()); + } CColRef *pcrProducer = (*pdrgpcrProducer)[remapUlPos]; pcrProducer->MarkAsUsed(); diff --git a/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp b/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp index 324e766f73d..1e3496e54e0 100644 --- a/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp @@ -2232,6 +2232,7 @@ CExpressionPreprocessor::CollectCTEPredicates(CMemoryPool *mp, CLogicalCTEProducer::PopConvert(pexprProducer->Pop()); UlongToColRefMap *colref_mapping = CUtils::PhmulcrMapping( mp, popConsumer->Pdrgpcr(), popProducer->Pdrgpcr()); + CExpression *pexprRemappedScalar = pexprScalar->PexprCopyWithRemappedColumns(mp, colref_mapping, true /*must_exist*/); @@ -3292,6 +3293,34 @@ CExpressionPreprocessor::ConvertSplitUpdateToInPlaceUpdate(CMemoryPool *mp, return pexpr; } +#if GPOS_DEBUG + +#define TRCAE_PREPROCESS_BEGIN(inpexpr) \ + if (GPOS_FTRACE(EopttracePrintPreProcessResult)) { \ + atio << "Begin pre-process. Input: " << std::endl; \ + inpexpr->OsPrint(atio) << std::endl; \ + } + + +#define TRCAE_PREPROCESS_STEP(output, desc) \ + if (GPOS_FTRACE(EopttracePrintPreProcessResult)) { \ + atio << "PreProcess(" << desc << ")"; \ + atio << " Output: " << std::endl; \ + output->OsPrint(atio) << std::endl; \ + } + +#define TRCAE_PREPROCESS_END() \ + if (GPOS_FTRACE(EopttracePrintPreProcessResult)) { \ + atio << "End pre-process." << std::endl; \ + } + +#else +#define TRCAE_PREPROCESS_BEGIN(inpexpr) +#define TRCAE_PREPROCESS_STEP(output, desc) +#define TRCAE_PREPROCESS_END() + +#endif // GPOS_DEBUG + // main driver, pre-processing of input logical expression CExpression * CExpressionPreprocessor::PexprPreprocess( @@ -3305,6 +3334,12 @@ CExpressionPreprocessor::PexprPreprocess( CAutoTimer at("\n[OPT]: Expression Preprocessing Time", GPOS_FTRACE(EopttracePrintOptimizationStatistics)); +#if GPOS_DEBUG + CAutoTrace atr(mp); + IOstream & atio = atr.Os(); +#endif + + TRCAE_PREPROCESS_BEGIN(pexpr); // remove unused CTE anchors CCTEInfo *pcteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); @@ -3312,22 +3347,29 @@ CExpressionPreprocessor::PexprPreprocess( CExpression *pexprNoUnusedCTEs = PexprRemoveUnusedCTEs(mp, pexpr); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprNoUnusedCTEs, "Remove unused CTEs"); // remove intermediate superfluous limit CExpression *pexprSimplifiedLimit = PexprRemoveSuperfluousLimit(mp, pexprNoUnusedCTEs); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprSimplifiedLimit, + "Remove intermediate superfluous limit"); pexprNoUnusedCTEs->Release(); // remove intermediate superfluous distinct CExpression *pexprSimplifiedDistinct = PexprRemoveSuperfluousDistinctInDQA(mp, pexprSimplifiedLimit); + TRCAE_PREPROCESS_STEP(pexprSimplifiedDistinct, + "Remove intermediate superfluous distinct"); GPOS_CHECK_ABORT; pexprSimplifiedLimit->Release(); // trim unnecessary existential subqueries CExpression *pexprTrimmed = PexprTrimExistentialSubqueries(mp, pexprSimplifiedDistinct); + TRCAE_PREPROCESS_STEP(pexprTrimmed, + "Trim unnecessary existential subqueries"); GPOS_CHECK_ABORT; pexprSimplifiedDistinct->Release(); @@ -3335,6 +3377,8 @@ CExpressionPreprocessor::PexprPreprocess( CExpression *pexprNaryUnionUnionAll = PexprCollapseUnionUnionAll(mp, pexprTrimmed); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprNaryUnionUnionAll, + "Collapse cascaded union / union all"); pexprTrimmed->Release(); // remove superfluous outer references from the order spec in limits, grouping columns in GbAgg, and @@ -3342,12 +3386,16 @@ CExpressionPreprocessor::PexprPreprocess( CExpression *pexprOuterRefsEleminated = PexprRemoveSuperfluousOuterRefs(mp, pexprNaryUnionUnionAll); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprOuterRefsEleminated, + "Remove superfluous outer references from the order spec in limits"); pexprNaryUnionUnionAll->Release(); // remove superfluous equality CExpression *pexprTrimmed2 = PexprPruneSuperfluousEquality(mp, pexprOuterRefsEleminated); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprTrimmed2, + "Remove superfluous equality"); pexprOuterRefsEleminated->Release(); // substitute constant predicates @@ -3355,10 +3403,12 @@ CExpressionPreprocessor::PexprPreprocess( CExpression *pexprPredWithConstReplaced = PexprReplaceColWithConst(mp, pexprTrimmed2, phmExprToConst, true); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprPredWithConstReplaced, + "substitute constant predicates"); phmExprToConst->Release(); pexprTrimmed2->Release(); - // reorder the children of scalar cmp operator to ensure that left + // reorder the children of scalar cmp operator to ensure that left // child is scalar ident and right child is scalar const // // Must happen after "substitute constant predicates" step which can insert scalar cmp children with inversed @@ -3367,24 +3417,32 @@ CExpressionPreprocessor::PexprPreprocess( CExpression *pexprReorderedScalarCmpChildren = PexprReorderScalarCmpChildren(mp, pexprPredWithConstReplaced); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprReorderedScalarCmpChildren, + "Reorder the children of scalar cmp operator to ensure that left child is scalar ident and right child is scalar const"); pexprPredWithConstReplaced->Release(); // simplify quantified subqueries CExpression *pexprSubqSimplified = PexprSimplifyQuantifiedSubqueries(mp, pexprReorderedScalarCmpChildren); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprSubqSimplified, + "Simplify quantified subqueries"); pexprReorderedScalarCmpChildren->Release(); // do preliminary unnesting of scalar subqueries CExpression *pexprSubqUnnested = PexprUnnestScalarSubqueries(mp, pexprSubqSimplified); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprSubqUnnested, + "Do preliminary unnesting of scalar subqueries"); pexprSubqSimplified->Release(); // unnest AND/OR/NOT predicates CExpression *pexprUnnested = CExpressionUtils::PexprUnnest(mp, pexprSubqUnnested); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprUnnested, + "Unnest AND/OR/NOT predicates"); pexprSubqUnnested->Release(); CExpression *pexprConvert2In = pexprUnnested; @@ -3405,117 +3463,158 @@ CExpressionPreprocessor::PexprPreprocess( CLeftJoinPruningPreprocessor::PexprPreprocess(mp, pexprConvert2In, pcrsOutputAndOrderCols); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprJoinPruned, + "Left Outer Join Pruning"); pexprConvert2In->Release(); // infer predicates from constraints CExpression *pexprInferredPreds = PexprInferPredicates(mp, pexprJoinPruned); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprInferredPreds, + "Infer predicates from constraints"); pexprJoinPruned->Release(); // eliminate self comparisons CExpression *pexprSelfCompEliminated = PexprEliminateSelfComparison( mp, pexprInferredPreds, pexprInferredPreds->DeriveNotNullColumns()); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprSelfCompEliminated, + "Eliminate self comparisons"); pexprInferredPreds->Release(); // remove duplicate AND/OR children CExpression *pexprDeduped = CExpressionUtils::PexprDedupChildren(mp, pexprSelfCompEliminated); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprDeduped, + "Remove duplicate AND/OR children"); pexprSelfCompEliminated->Release(); // factorize common expressions CExpression *pexprFactorized = CExpressionFactorizer::PexprFactorize(mp, pexprDeduped); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprFactorized, + "Factorize common expressions"); pexprDeduped->Release(); // infer filters out of components of disjunctive filters CExpression *pexprPrefiltersExtracted = CExpressionFactorizer::PexprExtractInferredFilters(mp, pexprFactorized); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprPrefiltersExtracted, + "Infer filters out of components of disjunctive filters"); pexprFactorized->Release(); // pre-process ordered agg functions CExpression *pexprOrderedAggPreprocessed = COrderedAggPreprocessor::PexprPreprocess(mp, pexprPrefiltersExtracted); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprOrderedAggPreprocessed, + "Pre-process ordered agg functions"); pexprPrefiltersExtracted->Release(); // eliminate unused computed columns CExpression *pexprNoUnusedPrEl = PexprPruneUnusedComputedCols( mp, pexprOrderedAggPreprocessed, pcrsOutputAndOrderCols); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprNoUnusedPrEl, + "Eliminate unused computed columns"); pexprOrderedAggPreprocessed->Release(); // normalize expression CExpression *pexprNormalized1 = CNormalizer::PexprNormalize(mp, pexprNoUnusedPrEl); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprNormalized1, + "Normalize expression"); pexprNoUnusedPrEl->Release(); // transform outer join into inner join whenever possible CExpression *pexprLOJToIJ = PexprOuterJoinToInnerJoin(mp, pexprNormalized1); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprLOJToIJ, + "Transform outer join into inner join whenever possible"); pexprNormalized1->Release(); // collapse cascaded inner and left outer joins CExpression *pexprCollapsed = PexprCollapseJoins(mp, pexprLOJToIJ); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprCollapsed, + "Collapse cascaded inner and left outer joins"); pexprLOJToIJ->Release(); // after transforming outer joins to inner joins, we may be able to generate more predicates from constraints CExpression *pexprWithPreds = PexprAddPredicatesFromConstraints(mp, pexprCollapsed); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprWithPreds, + "After transforming outer joins to inner joins, we may be able to generate more predicates from constraints"); pexprCollapsed->Release(); // eliminate empty subtrees CExpression *pexprPruned = PexprPruneEmptySubtrees(mp, pexprWithPreds); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprPruned, + "Eliminate empty subtrees"); pexprWithPreds->Release(); // collapse cascade of projects CExpression *pexprCollapsedProjects = PexprCollapseProjects(mp, pexprPruned); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprCollapsedProjects, + "Collapse cascade of projects"); pexprPruned->Release(); // insert dummy project when the scalar subquery is under a project and returns an outer reference CExpression *pexprSubquery = PexprProjBelowSubquery( mp, pexprCollapsedProjects, false /* fUnderPrList */); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprSubquery, + "Insert dummy project when the scalar subquery is under a project and returns an outer reference"); pexprCollapsedProjects->Release(); // rewrite IN subquery to EXIST subquery with a predicate CExpression *pexprExistWithPredFromINSubq = PexprExistWithPredFromINSubq(mp, pexprSubquery); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprExistWithPredFromINSubq, + "Rewrite IN subquery to EXIST subquery with a predicate"); pexprSubquery->Release(); // prune partitions CExpression *pexprPrunedPartitions = PrunePartitions(mp, pexprExistWithPredFromINSubq); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprPrunedPartitions, + "Prune partitions"); pexprExistWithPredFromINSubq->Release(); // swap logical select over logical project CExpression *pexprTransposeSelectAndProject = PexprTransposeSelectAndProject(mp, pexprPrunedPartitions); + TRCAE_PREPROCESS_STEP(pexprTransposeSelectAndProject, + "Swap logical select over logical project"); pexprPrunedPartitions->Release(); // convert split update to inplace update CExpression *pexprSplitUpdateToInplace = ConvertSplitUpdateToInPlaceUpdate(mp, pexprTransposeSelectAndProject); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprSplitUpdateToInplace, + "Convert split update to inplace update"); pexprTransposeSelectAndProject->Release(); // normalize expression again CExpression *pexprNormalized2 = CNormalizer::PexprNormalize(mp, pexprSplitUpdateToInplace); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprNormalized2, + "Normalize expression"); pexprSplitUpdateToInplace->Release(); + // join hint CPlanHint *planhint = COptCtxt::PoctxtFromTLS()->GetOptimizerConfig()->GetPlanHint(); if (nullptr != planhint) @@ -3525,11 +3624,15 @@ CExpressionPreprocessor::PexprPreprocess( CJoinOrderHintsPreprocessor::PexprPreprocess( mp, pexprNormalized2, nullptr /* joinnode */); GPOS_CHECK_ABORT; + TRCAE_PREPROCESS_STEP(pexprJoinHintsApplied, + "Apply join order hints"); pexprNormalized2->Release(); + TRCAE_PREPROCESS_END(); return pexprJoinHintsApplied; } + TRCAE_PREPROCESS_END(); return pexprNormalized2; } diff --git a/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp b/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp index d8d29700c0a..d1557f5454d 100644 --- a/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CLogicalCTEConsumer.cpp @@ -36,7 +36,8 @@ CLogicalCTEConsumer::CLogicalCTEConsumer(CMemoryPool *mp) m_pdrgpcr(nullptr), m_pexprInlined(nullptr), m_phmulcr(nullptr), - m_pcrsOutput(nullptr) + m_pcrsOutput(nullptr), + m_pruned(false) { m_fPattern = true; } @@ -55,7 +56,8 @@ CLogicalCTEConsumer::CLogicalCTEConsumer(CMemoryPool *mp, ULONG id, m_id(id), m_pdrgpcr(colref_array), m_pexprInlined(nullptr), - m_phmulcr(nullptr) + m_phmulcr(nullptr), + m_pruned(false) { GPOS_ASSERT(nullptr != colref_array); m_pcrsOutput = GPOS_NEW(mp) CColRefSet(mp, m_pdrgpcr); @@ -107,6 +109,38 @@ CLogicalCTEConsumer::CreateInlinedExpr(CMemoryPool *mp) mp, m_phmulcr, true /*must_exist*/); } +void +CLogicalCTEConsumer::ApplyInline() { + // do nothing + if (!m_pruned) { + return; + } + CCTEInfo *pcteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); + CExpressionArray *pcexprs = pcteinfo->PexprCTEConsumer(m_id); + ULONG cpexrssz = pcexprs->Size(); + UlongToColRefMap *pcmpulcr = pcteinfo->GetCTEConsumerMapping(); + + for (ULONG ulPos = 0; ulPos < cpexrssz; ulPos++) { + CExpression *cepxr = (*pcexprs)[ulPos]; + CLogicalCTEConsumer *pcoper = CLogicalCTEConsumer::PopConvert(cepxr->Pop()); + CColRefArray *colrefs = pcoper->Pdrgpcr(); + + for (ULONG ulcrPos = 0; ulcrPos < colrefs->Size(); ulcrPos++) { + CColRef *colref = (*colrefs)[ulcrPos]; + ULONG colid = colref->Id(); + colref->MarkAsUsed(); + + CColRef *producer_colref = nullptr; + producer_colref = pcmpulcr->Find(&colid); + if (producer_colref) { + producer_colref->MarkAsUsed(); + } + } + + pcoper->MarkAsNotPruned(); + } +} + //--------------------------------------------------------------------------- // @function: // CLogicalCTEConsumer::DeriveOutputColumns @@ -120,6 +154,7 @@ CLogicalCTEConsumer::DeriveOutputColumns(CMemoryPool *, //mp, CExpressionHandle & //exprhdl ) { + // It's fine if we not prune the pcrsOutput m_pcrsOutput->AddRef(); return m_pcrsOutput; } @@ -320,27 +355,24 @@ CLogicalCTEConsumer::PopCopyWithRemappedColumns( CMemoryPool *mp, UlongToColRefMap *colref_mapping, BOOL must_exist) { CColRefArray *colref_array = nullptr; - CCTEInfo * cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); + CCTEInfo *cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); if (must_exist) { UlongToColRefMap *consumer_mapping; - UlongToColRefArrayMap *producer_mapping; ULONG colid; ULONG colref_size; - colref_array = - CUtils::PdrgpcrRemapAndCreate(mp, m_pdrgpcr, colref_mapping); - + colref_array = CUtils::PdrgpcrRemapAndCreate(mp, m_pdrgpcr, colref_mapping); + colref_size = colref_array->Size(); GPOS_ASSERT(colref_size == colref_array->Size()); - + consumer_mapping = cteinfo->GetCTEConsumerMapping(); - producer_mapping = cteinfo->GetCTEProducerMapping(); for (ULONG ul = 0; ul < colref_size; ul++) { CColRef *old_colref = (*m_pdrgpcr)[ul]; CColRef *new_colref = (*colref_array)[ul]; - + colid = old_colref->Id(); CColRef *p_colref = consumer_mapping->Find(&colid); if (nullptr == p_colref) { @@ -349,25 +381,9 @@ CLogicalCTEConsumer::PopCopyWithRemappedColumns( GPOS_WSZ_LIT( "Not found CTE consumer colid regsiterd in consumer mapping")); } - - BOOL fInserted = consumer_mapping->Insert( + + consumer_mapping->Insert( GPOS_NEW(mp) ULONG(new_colref->Id()), p_colref); - - if (fInserted) { - colid = p_colref->Id(); - - const CColRefArray *crarray = producer_mapping->Find(&colid); - if (nullptr == crarray) - { - GPOS_RAISE( - CException::ExmaInvalid, CException::ExmiInvalid, - GPOS_WSZ_LIT( - "Not found CTE consumer colid regsiterd in consumer mapping")); - } - - (const_cast(crarray)) - ->Append(new_colref); - } } } else @@ -375,7 +391,12 @@ CLogicalCTEConsumer::PopCopyWithRemappedColumns( colref_array = CUtils::PdrgpcrRemap(mp, m_pdrgpcr, colref_mapping, must_exist); } - return GPOS_NEW(mp) CLogicalCTEConsumer(mp, m_id, colref_array); + + CExpression *new_pexpr = GPOS_NEW(mp) CExpression( + mp, GPOS_NEW(mp) CLogicalCTEConsumer(mp, m_id, colref_array)); + + cteinfo->AddCTEConsumer(new_pexpr); + return new_pexpr->Pop(); } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/operators/CLogicalCTEProducer.cpp b/src/backend/gporca/libgpopt/src/operators/CLogicalCTEProducer.cpp index c1bdcf0d2f2..a1f9a5420f4 100644 --- a/src/backend/gporca/libgpopt/src/operators/CLogicalCTEProducer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CLogicalCTEProducer.cpp @@ -27,7 +27,8 @@ using namespace gpopt; // //--------------------------------------------------------------------------- CLogicalCTEProducer::CLogicalCTEProducer(CMemoryPool *mp) - : CLogical(mp), m_id(0), m_pdrgpcr(nullptr), m_pcrsOutput(nullptr) + : CLogical(mp), m_id(0), m_pdrgpcr(nullptr), m_pcrsOutput(nullptr), + m_cbp(false), m_umask(nullptr) { m_fPattern = true; } @@ -41,8 +42,10 @@ CLogicalCTEProducer::CLogicalCTEProducer(CMemoryPool *mp) // //--------------------------------------------------------------------------- CLogicalCTEProducer::CLogicalCTEProducer(CMemoryPool *mp, ULONG id, - CColRefArray *colref_array) - : CLogical(mp), m_id(id), m_pdrgpcr(colref_array) + CColRefArray *colref_array, + BOOL canbepruned) + : CLogical(mp), m_id(id), m_pdrgpcr(colref_array), + m_pcrsOutput(nullptr), m_cbp(canbepruned), m_umask(nullptr) { GPOS_ASSERT(nullptr != colref_array); @@ -52,6 +55,22 @@ CLogicalCTEProducer::CLogicalCTEProducer(CMemoryPool *mp, ULONG id, m_pcrsLocalUsed->Include(m_pdrgpcr); } +void +CLogicalCTEProducer::RecalOutputColumns(BOOL *umask, ULONG sz) { + CRefCount::SafeRelease(m_pcrsOutput); + + GPOS_ASSERT(m_pdrgpcr->Size() == sz); + + m_pcrsOutput = GPOS_NEW(m_mp) CColRefSet(m_mp); + for (ULONG ul = 0; ul < sz; ul++) { + if (umask[ul]) { + m_pcrsOutput->Include((*m_pdrgpcr)[ul]); + } + } + + m_umask = umask; +} + //--------------------------------------------------------------------------- // @function: // CLogicalCTEProducer::~CLogicalCTEProducer @@ -64,6 +83,9 @@ CLogicalCTEProducer::~CLogicalCTEProducer() { CRefCount::SafeRelease(m_pdrgpcr); CRefCount::SafeRelease(m_pcrsOutput); + if (m_umask) { + GPOS_DELETE_ARRAY(m_umask); + } } //--------------------------------------------------------------------------- @@ -95,7 +117,8 @@ CColRefSet * CLogicalCTEProducer::DeriveNotNullColumns(CMemoryPool *mp, CExpressionHandle &exprhdl) const { - CColRefSet *pcrs = GPOS_NEW(mp) CColRefSet(mp, m_pdrgpcr); + CColRefSet *pcrs = GPOS_NEW(mp) CColRefSet(mp); + pcrs->Include(m_pcrsOutput); pcrs->Intersection(exprhdl.DeriveNotNullColumns(0)); return pcrs; @@ -162,7 +185,8 @@ CLogicalCTEProducer::Matches(COperator *pop) const CLogicalCTEProducer *popCTEProducer = CLogicalCTEProducer::PopConvert(pop); return m_id == popCTEProducer->UlCTEId() && - m_pdrgpcr->Equals(popCTEProducer->Pdrgpcr()); + m_pdrgpcr->Equals(popCTEProducer->Pdrgpcr()) && + m_cbp == popCTEProducer->CanBePruned(); } //--------------------------------------------------------------------------- @@ -178,6 +202,7 @@ CLogicalCTEProducer::HashValue() const { ULONG ulHash = gpos::CombineHashes(COperator::HashValue(), m_id); ulHash = gpos::CombineHashes(ulHash, CUtils::UlHashColArray(m_pdrgpcr)); + ulHash = gpos::CombineHashes(ulHash, (ULONG)m_cbp); return ulHash; } diff --git a/src/backend/gporca/libgpopt/src/operators/COrderedAggPreprocessor.cpp b/src/backend/gporca/libgpopt/src/operators/COrderedAggPreprocessor.cpp index ada9508a6f0..ed1c3420697 100644 --- a/src/backend/gporca/libgpopt/src/operators/COrderedAggPreprocessor.cpp +++ b/src/backend/gporca/libgpopt/src/operators/COrderedAggPreprocessor.cpp @@ -475,11 +475,13 @@ COrderedAggPreprocessor::CreateCTE(CMemoryPool *mp, CExpression *pexprChild, mp, GPOS_NEW(mp) CLogicalCTEConsumer( mp, ulCTEId, CUtils::PdrgpcrCopy(mp, pdrgpcrProducerOutput))); pcteinfo->IncrementConsumers(ulCTEId); + pcteinfo->AddCTEConsumer(*ppexprFirstConsumer); pdrgpcrProducerOutput->Release(); // second consumer reuses the same output columns of SeqPrj child to be able to provide any requested columns upstream *ppexprSecondConsumer = GPOS_NEW(mp) CExpression( mp, GPOS_NEW(mp) CLogicalCTEConsumer(mp, ulCTEId, pdrgpcrChildOutput)); + pcteinfo->AddCTEConsumer(*ppexprSecondConsumer); pcteinfo->IncrementConsumers(ulCTEId); } diff --git a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp index fd593ef523e..d7a51950fa4 100644 --- a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEConsumer.cpp @@ -35,19 +35,27 @@ CPhysicalCTEConsumer::CPhysicalCTEConsumer(CMemoryPool *mp, ULONG id, : CPhysical(mp), m_id(id), m_pdrgpcr(nullptr), - m_phmulcr(colref_mapping) + m_phmulcr(colref_mapping), + m_pidxmap(nullptr) { GPOS_ASSERT(nullptr != colref_array); GPOS_ASSERT(nullptr != colref_mapping); ULONG colref_size = colref_array->Size(); m_pdrgpcr = GPOS_NEW(mp) CColRefArray(mp); + m_pidxmap = GPOS_NEW(mp) ULongPtrArray(mp); for (ULONG index = 0; index < colref_size; index++) { CColRef *col_ref = (*colref_array)[index]; - if (col_ref->GetUsage() == CColRef::EUsed) { + if (col_ref->GetUsage(true, true) == CColRef::EUsed) { m_pdrgpcr->Append(col_ref); - } + m_pidxmap->Append(GPOS_NEW(m_mp) ULONG(index)); + } + } + + if (m_pidxmap->Size() == colref_size) { + m_pidxmap->Release(); + m_pidxmap = nullptr; } } @@ -63,6 +71,7 @@ CPhysicalCTEConsumer::~CPhysicalCTEConsumer() { m_pdrgpcr->Release(); m_phmulcr->Release(); + CRefCount::SafeRelease(m_pidxmap); } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp index d44877b4f54..c6be21f87cd 100644 --- a/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CPhysicalCTEProducer.cpp @@ -30,26 +30,48 @@ using namespace gpopt; // //--------------------------------------------------------------------------- CPhysicalCTEProducer::CPhysicalCTEProducer(CMemoryPool *mp, ULONG id, - CColRefArray *colref_array) - : CPhysical(mp), m_id(id), m_pdrgpcr(nullptr), m_pcrs(nullptr) + CColRefArray *colref_array, + BOOL *umask) + : CPhysical(mp), m_id(id), m_pdrgpcr(nullptr), m_pcrs(nullptr), m_pdrgpcr_unused(nullptr), m_pidxmap(nullptr) { GPOS_ASSERT(nullptr != colref_array); - ULONG colref_size = colref_array->Size(); - ULONG unused_inc = 0; - m_pcrs = GPOS_NEW(mp) CColRefSet(mp); - m_pdrgpcr = GPOS_NEW(mp) CColRefArray(mp); - m_pdrgpcr_unused = GPOS_NEW(mp) ULongPtrArray(mp, colref_size); - - for (ULONG index = 0; index < colref_size; index++) { - CColRef *col_ref = (*colref_array)[index]; - GPOS_ASSERT(col_ref->GetUsage() != CColRef::EUnknown); - m_pdrgpcr_unused->Append(GPOS_NEW(m_mp) ULONG(unused_inc)); - if (col_ref->GetUsage() == CColRef::EUsed) { - m_pdrgpcr->Append(col_ref); - m_pcrs->Include(col_ref); - } else { - unused_inc++; + +#ifdef GPOS_DEBUG + if (umask) { + ULONG colref_size = colref_array->Size(); + for (ULONG ul = 0; ul < colref_size; ul++) { + CColRef *col_ref = (*colref_array)[ul]; + if (col_ref->GetUsage(true, true) != CColRef::EUsed) { + GPOS_ASSERT(!umask[ul]); + } + } + } +#endif + + if (umask) { + ULONG colref_size = colref_array->Size(); + ULONG unused_inc = 0; + m_pcrs = GPOS_NEW(mp) CColRefSet(mp); + m_pdrgpcr = GPOS_NEW(mp) CColRefArray(mp); + m_pdrgpcr_unused = GPOS_NEW(mp) ULongPtrArray(mp, colref_size); + m_pidxmap = GPOS_NEW(mp) ULongPtrArray(mp, colref_size); + + for (ULONG ul = 0; ul < colref_size; ul++) { + CColRef *col_ref = (*colref_array)[ul]; + m_pdrgpcr_unused->Append(GPOS_NEW(m_mp) ULONG(unused_inc)); + if (umask[ul]) { + m_pidxmap->Append(GPOS_NEW(m_mp) ULONG(m_pdrgpcr->Size())); + m_pdrgpcr->Append(col_ref); + m_pcrs->Include(col_ref); + } else { + m_pidxmap->Append(GPOS_NEW(m_mp) ULONG(gpos::ulong_max)); + unused_inc++; + } } + + } else { + m_pdrgpcr = colref_array; + m_pcrs = GPOS_NEW(mp) CColRefSet(mp, m_pdrgpcr); } } @@ -65,7 +87,8 @@ CPhysicalCTEProducer::~CPhysicalCTEProducer() { m_pdrgpcr->Release(); m_pcrs->Release(); - m_pdrgpcr_unused->Release(); + CRefCount::SafeRelease(m_pdrgpcr_unused); + CRefCount::SafeRelease(m_pidxmap); } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp index 1d7851855c4..0339db778b9 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorDXLToExpr.cpp @@ -18,6 +18,7 @@ #include "gpopt/base/CAutoOptCtxt.h" #include "gpopt/base/CColRef.h" #include "gpopt/base/CColRefSet.h" +#include "gpopt/base/CColRefSetIter.h" #include "gpopt/base/CColumnFactory.h" #include "gpopt/base/CDistributionSpecAny.h" #include "gpopt/base/CEnfdDistribution.h" @@ -292,9 +293,105 @@ CTranslatorDXLToExpr::Pexpr(const CDXLNode *dxlnode, } } + // Finally, prune the unused output in CTE producer and consumer + (void) PruneCTEs(); return pexpr; } +//--------------------------------------------------------------------------- +// @function: +// CTranslatorDXLToExpr::Pexpr +// +// @doc: +// prune the CTEs producer and consumer after DXL tree translated +// +//--------------------------------------------------------------------------- +void CTranslatorDXLToExpr::PruneCTEs() +{ + CCTEInfo *pcteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); + ULONG group_of_ctes = pcteinfo->CTESize(); + if (group_of_ctes > 0) { + for (ULONG ulCTEId = 0; ulCTEId < group_of_ctes; ulCTEId++){ + BOOL *umask; + + CExpression *ppexpr = pcteinfo->PexprCTEProducer(ulCTEId); + CLogicalCTEProducer *poper = CLogicalCTEProducer::PopConvert(ppexpr->Pop()); + ULONG pprgpcrsz = poper->Pdrgpcr()->Size(); + + // the producer can't be prune + if (!poper->CanBePruned()) { + continue; + } + + CExpressionArray *cexprs = pcteinfo->PexprCTEConsumer(ulCTEId); + ULONG cpexrssz = cexprs->Size(); + + // The current consumer is likely to be inlined + // So just not prune it. + if (pcteinfo->UlConsumers(poper->UlCTEId()) == 1) { + for (ULONG ulPos = 0; ulPos < cpexrssz; ulPos++) { + CExpression *cepxr = (*cexprs)[ulPos]; + CLogicalCTEConsumer *coper = CLogicalCTEConsumer::PopConvert(cepxr->Pop()); + CColRefArray *colrefs = coper->Pdrgpcr(); + for (ULONG ulcrPos = 0; ulcrPos < colrefs->Size(); ulcrPos++) { + CColRef *colref = (*colrefs)[ulcrPos]; + colref->MarkAsUsed(); + MarkCTEConsumerColAsUsed(pcteinfo->GetCTEConsumerMapping(), colref->Id()); + } + } + continue; + } + + // Calculate the columns of the current consumer requested + umask = GPOS_NEW_ARRAY(m_mp, BOOL, pprgpcrsz); + clib::Memset(umask, false, pprgpcrsz * sizeof(BOOL)); + + for (ULONG ulPos = 0; ulPos < cpexrssz; ulPos++) { + CExpression *cepxr = (*cexprs)[ulPos]; + CLogicalCTEConsumer *coper = CLogicalCTEConsumer::PopConvert(cepxr->Pop()); + CColRefArray *colrefs = coper->Pdrgpcr(); + + GPOS_ASSERT(pprgpcrsz == colrefs->Size()); + + for (ULONG ulcrPos = 0; ulcrPos < pprgpcrsz; ulcrPos++) { + if ((*colrefs)[ulcrPos]->GetUsage(true, true) == CColRef::EUsed) { + umask[ulcrPos] = true; + } + } + + coper->MarkAsPruned(); + } + + // Recalculate the producer's output columns + // no need relase umask + poper->RecalOutputColumns(umask, pprgpcrsz); + + // Align consumer and producer output columns. + // In fact, we can support the column projection in consumer(SharedInputScan). + // However, non-requested columns in consumer(which from producer) may be added to the consumer, + // which will make the plan for consumer(support projection) very complicated. + // + // In addition, the performance gain is not significant when the consumer supports projection. + CColRefSet *pcrs = poper->DeriveOutputColumns(); + for (ULONG ulPos = 0; ulPos < cpexrssz; ulPos++) { + CExpression *cepxr = (*cexprs)[ulPos]; + CLogicalCTEConsumer *coper = CLogicalCTEConsumer::PopConvert(cepxr->Pop()); + UlongToColRefMap *colmaping = coper->Phmulcr(); + + CColRefSetIter crsi(*pcrs); + while (crsi.Advance()) + { + ULONG colid = crsi.Pcr()->Id(); + CColRef *consumer_colref = colmaping->Find(&colid); + GPOS_ASSERT(consumer_colref); + consumer_colref->MarkAsUsed(); + } + + } + } + } +} + //--------------------------------------------------------------------------- // @function: // CTranslatorDXLToExpr::Pexpr @@ -1030,70 +1127,21 @@ CTranslatorDXLToExpr::FCastingUnknownType(IMDId *pmdidSource, IMDId *mdid_dest) mdid_dest->Equals(&CMDIdGPDB::m_mdid_unknown))); } +//--------------------------------------------------------------------------- +// @function: +// CTranslatorDXLToExpr::MarkCTEConsumerColAsUsed +// +// @doc: +// after the colref in CTE consumer marked used, the producer should marked +// the relatived colref as used. +//--------------------------------------------------------------------------- void -CTranslatorDXLToExpr::MarkCTEConsumerColAsUsed(UlongToColRefMap *mcidcrCTE, UlongToColRefArrayMap *mpidcrsCTE, ULONG colid) { +CTranslatorDXLToExpr::MarkCTEConsumerColAsUsed(UlongToColRefMap *mcidcrCTE, ULONG colid) { CColRef *producer_colref = nullptr; producer_colref = mcidcrCTE->Find(&colid); if (producer_colref) { - -#ifdef GPOS_DEBUG - if (GPOS_FTRACE(EopttraceDebugCTE)) { - CAutoTrace at(m_mp); - auto & io_stream = at.Os(); - io_stream << "MarkProducerUsed, customer id: " << colid << ", producer id:" << producer_colref->Id() << std::endl; - } -#endif - if (producer_colref->GetUsage(true, true) != CColRef::EUsed) { - producer_colref->MarkAsUsed(); - MarkCTEProducerColAsUsed(mcidcrCTE, mpidcrsCTE, producer_colref->Id()); - } - } -} - -void -CTranslatorDXLToExpr::MarkCTEProducerColAsUsed(UlongToColRefMap *mcidcrCTE, UlongToColRefArrayMap *mpidcrsCTE, ULONG colid) { - CColRefArray *consumer_crarray = nullptr; - -#ifdef GPOS_DEBUG - CAutoTrace at(m_mp); - auto & io_stream = at.Os(); - if (GPOS_FTRACE(EopttraceDebugCTE)) - io_stream << "MarkCTEProducerColAsUsed, producer id: " << colid; -#endif - - consumer_crarray = mpidcrsCTE->Find(&colid); - if (consumer_crarray) { - -#ifdef GPOS_DEBUG - if (GPOS_FTRACE(EopttraceDebugCTE)) - io_stream << ", consumer ids: ["; -#endif - - ULONG crarray_size = consumer_crarray->Size(); - for (ULONG crindex = 0; crindex < crarray_size; crindex++){ - auto consumer_colref = (*consumer_crarray)[crindex]; - -#ifdef GPOS_DEBUG - if (GPOS_FTRACE(EopttraceDebugCTE)) - io_stream << (*consumer_crarray)[crindex]->Id() << ","; -#endif - - if (consumer_colref->GetUsage(true, true) != CColRef::EUsed) { - consumer_colref->MarkAsUsed(); - MarkCTEConsumerColAsUsed(mcidcrCTE, mpidcrsCTE, consumer_colref->Id()); - } - } - -#ifdef GPOS_DEBUG - if (GPOS_FTRACE(EopttraceDebugCTE)) - io_stream << "]"; -#endif + producer_colref->MarkAsUsed(); } -#ifdef GPOS_DEBUG - else if (GPOS_FTRACE(EopttraceDebugCTE)) { - io_stream << ",not found consumer."; - } -#endif } //--------------------------------------------------------------------------- @@ -1116,11 +1164,10 @@ CTranslatorDXLToExpr::LookupColRef(ULONG colid, BOOL mark_used) GPOS_RAISE(gpdxl::ExmaDXL, gpdxl::ExmiDXL2ExprAttributeNotFound, colid); } - auto cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); + auto pcteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); - if (cteinfo->ExistCTE()) { - MarkCTEConsumerColAsUsed(cteinfo->GetCTEConsumerMapping(), cteinfo->GetCTEProducerMapping(), colref->Id()); - MarkCTEProducerColAsUsed(cteinfo->GetCTEConsumerMapping(), cteinfo->GetCTEProducerMapping(), colref->Id()); + if (pcteinfo->ExistCTE()) { + MarkCTEConsumerColAsUsed(pcteinfo->GetCTEConsumerMapping(), colref->Id()); } if (mark_used) @@ -1384,6 +1431,7 @@ CTranslatorDXLToExpr::PexprLogicalCTEProducer(const CDXLNode *dxlnode) CDXLLogicalCTEProducer *pdxlopCTEProducer = CDXLLogicalCTEProducer::Cast(dxlnode->GetOperator()); ULONG id = UlMapCTEId(pdxlopCTEProducer->Id()); + BOOL cbp = pdxlopCTEProducer->CouldBePruned(); // translate the child dxl node CExpression *pexprChild = PexprLogical((*dxlnode)[0]); @@ -1411,7 +1459,7 @@ CTranslatorDXLToExpr::PexprLogicalCTEProducer(const CDXLNode *dxlnode) for (ULONG ul = 0; ul < length; ul++) { ULONG *pulColId = (*pdrgpulCols)[ul]; - CColRef *colref = LookupColRef(*pulColId, !pdxlopCTEProducer->CouldBePruned() /* mark_used */); + CColRef *colref = LookupColRef(*pulColId, !cbp /* mark_used */); GPOS_ASSERT(nullptr != colref); #ifdef GPOS_DEBUG @@ -1455,7 +1503,7 @@ CTranslatorDXLToExpr::PexprLogicalCTEProducer(const CDXLNode *dxlnode) pcrsProducer->Release(); return GPOS_NEW(m_mp) CExpression( - m_mp, GPOS_NEW(m_mp) CLogicalCTEProducer(m_mp, id, colref_array), + m_mp, GPOS_NEW(m_mp) CLogicalCTEProducer(m_mp, id, colref_array, cbp), pexprChild); } @@ -1482,12 +1530,12 @@ CTranslatorDXLToExpr::PexprLogicalCTEConsumer(const CDXLNode *dxlnode) CCTEInfo *pcteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); CExpression *pexprProducer = pcteinfo->PexprCTEProducer(id); GPOS_ASSERT(nullptr != pexprProducer); + CLogicalCTEProducer *popProducer= CLogicalCTEProducer::PopConvert(pexprProducer->Pop()); - CCTEInfo * cteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); - CColRefArray *pdrgpcrProducer = - CLogicalCTEProducer::PopConvert(pexprProducer->Pop())->Pdrgpcr(); + CColRefArray *pdrgpcrProducer = popProducer->Pdrgpcr(); + BOOL can_pruned = popProducer->CanBePruned(); CColRefArray *pdrgpcrConsumer = CUtils::PdrgpcrCopy(m_mp, pdrgpcrProducer, - false, nullptr, cteinfo->GetCTEConsumerMapping(), cteinfo->GetCTEProducerMapping(), true); + false, nullptr, pcteinfo->GetCTEConsumerMapping(), nullptr, can_pruned); // add new colrefs to mapping const ULONG num_cols = pdrgpcrConsumer->Size(); @@ -1497,6 +1545,10 @@ CTranslatorDXLToExpr::PexprLogicalCTEConsumer(const CDXLNode *dxlnode) ULONG *pulColId = GPOS_NEW(m_mp) ULONG(*(*pdrgpulCols)[ul]); CColRef *colref = (*pdrgpcrConsumer)[ul]; + // reset consumer col array to unknown + if (can_pruned) + colref->MarkAsUnknown(); + BOOL result GPOS_ASSERTS_ONLY = m_phmulcr->Insert(pulColId, colref); GPOS_ASSERT(result); @@ -1521,9 +1573,12 @@ CTranslatorDXLToExpr::PexprLogicalCTEConsumer(const CDXLNode *dxlnode) #endif } - pcteinfo->IncrementConsumers(id, m_ulCTEId); - return GPOS_NEW(m_mp) CExpression( + CExpression *pexprcs = GPOS_NEW(m_mp) CExpression( m_mp, GPOS_NEW(m_mp) CLogicalCTEConsumer(m_mp, id, pdrgpcrConsumer)); + pcteinfo->IncrementConsumers(id, m_ulCTEId); + pcteinfo->AddCTEConsumer(pexprcs); + + return pexprcs; } //--------------------------------------------------------------------------- diff --git a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp index 07b6b54ecc5..d213bd1c0a1 100644 --- a/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp +++ b/src/backend/gporca/libgpopt/src/translate/CTranslatorExprToDXL.cpp @@ -2399,8 +2399,13 @@ CTranslatorExprToDXL::PdxlnCTEProducer( CDXLNode *pdxlnPrL = PdxlnProjList(pcrsOutput, pdrgpcrRequired); pcrsOutput->Release(); + ULongPtrArray *producer_idx_map = popCTEProducer->PCTEIdxMap(); + if (producer_idx_map) { + producer_idx_map->AddRef(); + } + CDXLNode *pdxlnCTEProducer = GPOS_NEW(m_mp) CDXLNode( - m_mp, GPOS_NEW(m_mp) CDXLPhysicalCTEProducer(m_mp, ulCTEId, colids), + m_mp, GPOS_NEW(m_mp) CDXLPhysicalCTEProducer(m_mp, ulCTEId, colids, producer_idx_map), pdxlnPrL, child_dxlnode); pdxlnCTEProducer->SetProperties(dxl_properties); @@ -2444,9 +2449,13 @@ CTranslatorExprToDXL::PdxlnCTEConsumer( // translate relational child expression CDXLNode *pdxlnPrL = PdxlnProjList(pcrsOutput, colref_array); + ULongPtrArray *pidxmap = popCTEConsumer->PCTEIdxMap(); + if (pidxmap) { + pidxmap->AddRef(); + } CDXLNode *pdxlnCTEConsumer = GPOS_NEW(m_mp) CDXLNode( - m_mp, GPOS_NEW(m_mp) CDXLPhysicalCTEConsumer(m_mp, ulCTEId, colids), + m_mp, GPOS_NEW(m_mp) CDXLPhysicalCTEConsumer(m_mp, ulCTEId, colids, pidxmap), pdxlnPrL); pcrsOutput->Release(); diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformExpandFullOuterJoin.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformExpandFullOuterJoin.cpp index 82d69dfb7e3..0f2b274a3e5 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformExpandFullOuterJoin.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformExpandFullOuterJoin.cpp @@ -95,17 +95,18 @@ CXformExpandFullOuterJoin::Transform(CXformContext *pxfctxt, GPOS_ASSERT(FCheckPattern(pexpr)); CMemoryPool *mp = pxfctxt->Pmp(); + CCTEInfo *pcteinfo = COptCtxt::PoctxtFromTLS()->Pcteinfo(); CExpression *pexprA = (*pexpr)[0]; CExpression *pexprB = (*pexpr)[1]; CExpression *pexprScalar = (*pexpr)[2]; // 1. create the CTE producers - const ULONG ulCTEIdA = COptCtxt::PoctxtFromTLS()->Pcteinfo()->next_id(); + const ULONG ulCTEIdA = pcteinfo->next_id(); CColRefArray *pdrgpcrOutA = pexprA->DeriveOutputColumns()->Pdrgpcr(mp); (void) CXformUtils::PexprAddCTEProducer(mp, ulCTEIdA, pdrgpcrOutA, pexprA); - const ULONG ulCTEIdB = COptCtxt::PoctxtFromTLS()->Pcteinfo()->next_id(); + const ULONG ulCTEIdB = pcteinfo->next_id(); CColRefArray *pdrgpcrOutB = pexprB->DeriveOutputColumns()->Pdrgpcr(mp); (void) CXformUtils::PexprAddCTEProducer(mp, ulCTEIdB, pdrgpcrOutB, pexprB); @@ -194,11 +195,13 @@ CXformExpandFullOuterJoin::PexprLogicalJoinOverCTEs( GPOS_NEW(mp) CLogicalCTEConsumer(mp, ulLeftCTEId, pdrgpcrLeft); CExpression *pexprLeft = GPOS_NEW(mp) CExpression(mp, popConsumerLeft); pcteinfo->IncrementConsumers(ulLeftCTEId); + pcteinfo->AddCTEConsumer(pexprLeft); CLogicalCTEConsumer *popConsumerRight = GPOS_NEW(mp) CLogicalCTEConsumer(mp, ulRightCTEId, pdrgpcrRight); CExpression *pexprRight = GPOS_NEW(mp) CExpression(mp, popConsumerRight); pcteinfo->IncrementConsumers(ulRightCTEId); + pcteinfo->AddCTEConsumer(pexprRight); pdrgpexprChildren->Append(pexprLeft); pdrgpexprChildren->Append(pexprRight); diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformGbAggWithMDQA2Join.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformGbAggWithMDQA2Join.cpp index 14041f41dc2..0ca823a499c 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformGbAggWithMDQA2Join.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformGbAggWithMDQA2Join.cpp @@ -113,6 +113,7 @@ CXformGbAggWithMDQA2Join::PexprMDQAs2Join(CMemoryPool *mp, CExpression *pexpr) CExpression *pexprConsumer = GPOS_NEW(mp) CExpression( mp, GPOS_NEW(mp) CLogicalCTEConsumer(mp, ulCTEId, pdrgpcrChildOutput)); pcteinfo->IncrementConsumers(ulCTEId); + pcteinfo->AddCTEConsumer(pexprConsumer); // finalize GbAgg expression by replacing its child with CTE consumer pexpr->Pop()->AddRef(); diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformImplementCTEProducer.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformImplementCTEProducer.cpp index baccad23b76..a9151baba18 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformImplementCTEProducer.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformImplementCTEProducer.cpp @@ -86,7 +86,7 @@ CXformImplementCTEProducer::Transform(CXformContext *pxfctxt, // create physical CTE Producer CExpression *pexprAlt = GPOS_NEW(mp) - CExpression(mp, GPOS_NEW(mp) CPhysicalCTEProducer(mp, id, colref_array), + CExpression(mp, GPOS_NEW(mp) CPhysicalCTEProducer(mp, id, colref_array, popCTEProducer->UsedMask()), pexprChild); // add alternative to transformation result diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumer.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumer.cpp index 2ad700dfa05..f03efdacf89 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumer.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumer.cpp @@ -84,6 +84,7 @@ CXformInlineCTEConsumer::Transform(CXformContext * // inline the consumer CLogicalCTEConsumer *popConsumer = CLogicalCTEConsumer::PopConvert(pexpr->Pop()); + (void) popConsumer->ApplyInline(); CExpression *pexprAlt = popConsumer->PexprInlined(); pexprAlt->AddRef(); // add alternative to xform result diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumerUnderSelect.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumerUnderSelect.cpp index 509355d14e6..8b1ddfc7117 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumerUnderSelect.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformInlineCTEConsumerUnderSelect.cpp @@ -103,6 +103,7 @@ CXformInlineCTEConsumerUnderSelect::Transform(CXformContext *pxfctxt, // inline consumer GPOS_ASSERT(nullptr != popConsumer->Phmulcr()); + (void) popConsumer->ApplyInline(); CExpression *pexprInlinedConsumer = popConsumer->PexprInlined(); pexprInlinedConsumer->AddRef(); pexprScalar->AddRef(); diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformUnnestTVF.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformUnnestTVF.cpp index 1623c81de94..d7eaa60e118 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformUnnestTVF.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformUnnestTVF.cpp @@ -222,6 +222,7 @@ CXformUnnestTVF::Transform(CXformContext *pxfctxt, CXformResult *pxfres, GPOS_NEW(mp) CLogicalCTEConsumer(mp, ulCTEId, pdrgpcrConsumerOutput); CExpression *pexprCTECons = GPOS_NEW(mp) CExpression(mp, popConsumer); pcteinfo->IncrementConsumers(ulCTEId); + pcteinfo->AddCTEConsumer(pexprCTECons); pdrgpcrProducerOutput->Release(); // find columns corresponding to subqueries in consumer's output diff --git a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp index cce598d386b..a21e3d963de 100644 --- a/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp +++ b/src/backend/gporca/libgpopt/src/xforms/CXformUtils.cpp @@ -3652,11 +3652,14 @@ CExpression * CXformUtils::PexprCTEConsumer(CMemoryPool *mp, ULONG ulCTEId, CColRefArray *colref_array) { + CExpression *pexpr; CLogicalCTEConsumer *popConsumer = GPOS_NEW(mp) CLogicalCTEConsumer(mp, ulCTEId, colref_array); - COptCtxt::PoctxtFromTLS()->Pcteinfo()->IncrementConsumers(ulCTEId); - return GPOS_NEW(mp) CExpression(mp, popConsumer); + pexpr = GPOS_NEW(mp) CExpression(mp, popConsumer); + COptCtxt::PoctxtFromTLS()->Pcteinfo()->IncrementConsumers(ulCTEId); + COptCtxt::PoctxtFromTLS()->Pcteinfo()->AddCTEConsumer(pexpr); + return pexpr; } @@ -4050,6 +4053,7 @@ CXformUtils::PexprGbAggOnCTEConsumer2Join(CMemoryPool *mp, CExpression(mp, GPOS_NEW(mp) CLogicalCTEConsumer( mp, ulCTEId, pdrgpcrNewConsumerOutput)); pcteinfo->IncrementConsumers(ulCTEId); + pcteinfo->AddCTEConsumer(pexprNewConsumer); // fix Aggs arguments to use new consumer output column UlongToColRefMap *colref_mapping = CUtils::PhmulcrMapping( diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEConsumer.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEConsumer.h index d17ddf7af53..dc6ae75ee2e 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEConsumer.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEConsumer.h @@ -35,12 +35,16 @@ class CDXLPhysicalCTEConsumer : public CDXLPhysical // output column ids ULongPtrArray *m_output_colids_array; + // output column index mapping + ULongPtrArray *m_output_colidx_map; + public: CDXLPhysicalCTEConsumer(CDXLPhysicalCTEConsumer &) = delete; // ctor CDXLPhysicalCTEConsumer(CMemoryPool *mp, ULONG id, - ULongPtrArray *output_colids_array); + ULongPtrArray *output_colids_array, + ULongPtrArray *output_colidx_map); // dtor ~CDXLPhysicalCTEConsumer() override; @@ -64,6 +68,12 @@ class CDXLPhysicalCTEConsumer : public CDXLPhysical return m_output_colids_array; } + ULongPtrArray * + GetOutputColIdxMap() const + { + return m_output_colidx_map; + } + // serialize operator in DXL format void SerializeToDXL(CXMLSerializer *xml_serializer, const CDXLNode *dxlnode) const override; diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEProducer.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEProducer.h index e03b4889451..9d8735e9b94 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEProducer.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/operators/CDXLPhysicalCTEProducer.h @@ -35,12 +35,16 @@ class CDXLPhysicalCTEProducer : public CDXLPhysical // output column ids ULongPtrArray *m_output_colids_array; + // output column index mapping + ULongPtrArray *m_output_colidx_map; + public: CDXLPhysicalCTEProducer(CDXLPhysicalCTEProducer &) = delete; // ctor CDXLPhysicalCTEProducer(CMemoryPool *mp, ULONG id, - ULongPtrArray *output_colids_array); + ULongPtrArray *output_colids_array, + ULongPtrArray *output_colidx_map); // dtor ~CDXLPhysicalCTEProducer() override; @@ -64,6 +68,13 @@ class CDXLPhysicalCTEProducer : public CDXLPhysical return m_output_colids_array; } + + ULongPtrArray * + GetOutputColIdxMap() const + { + return m_output_colidx_map; + } + // serialize operator in DXL format void SerializeToDXL(CXMLSerializer *xml_serializer, const CDXLNode *dxlnode) const override; diff --git a/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h b/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h index 9c21027c05b..e682f96a6d3 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/dxl/xml/dxltokens.h @@ -354,6 +354,7 @@ enum Edxltoken EdxltokenColDescr, EdxltokenColRef, + EdxltokenColIdxmap, EdxltokenColumns, EdxltokenColumn, diff --git a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h index 684fcd60b26..82f7ffb16b6 100644 --- a/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h +++ b/src/backend/gporca/libnaucrates/include/naucrates/traceflags/traceflags.h @@ -77,8 +77,11 @@ enum EOptTraceFlag // log results of hint parsing EopttracePrintPgHintPlanLog = 101018, + // print result of preprocess + EopttracePrintPreProcessResult = 101019, + // print debug info of CTE - EopttraceDebugCTE = 101019, + EopttraceDebugCTE = 101020, /////////////////////////////////////////////////////// ////////////////// transformations flags ////////////// diff --git a/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEConsumer.cpp b/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEConsumer.cpp index c95935965a7..67271473087 100644 --- a/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEConsumer.cpp +++ b/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEConsumer.cpp @@ -30,8 +30,8 @@ using namespace gpdxl; // //--------------------------------------------------------------------------- CDXLPhysicalCTEConsumer::CDXLPhysicalCTEConsumer( - CMemoryPool *mp, ULONG id, ULongPtrArray *output_colids_array) - : CDXLPhysical(mp), m_id(id), m_output_colids_array(output_colids_array) + CMemoryPool *mp, ULONG id, ULongPtrArray *output_colids_array, ULongPtrArray *output_colidx_map) + : CDXLPhysical(mp), m_id(id), m_output_colids_array(output_colids_array), m_output_colidx_map(output_colidx_map) { GPOS_ASSERT(nullptr != output_colids_array); } @@ -47,6 +47,7 @@ CDXLPhysicalCTEConsumer::CDXLPhysicalCTEConsumer( CDXLPhysicalCTEConsumer::~CDXLPhysicalCTEConsumer() { m_output_colids_array->Release(); + CRefCount::SafeRelease(m_output_colidx_map); } //--------------------------------------------------------------------------- @@ -102,6 +103,15 @@ CDXLPhysicalCTEConsumer::SerializeToDXL(CXMLSerializer *xml_serializer, str_colids); GPOS_DELETE(str_colids); + if (m_output_colidx_map) { + CWStringDynamic *str_colidx_map = + CDXLUtils::Serialize(m_mp, m_output_colidx_map); + + xml_serializer->AddAttribute(CDXLTokens::GetDXLTokenStr(EdxltokenColIdxmap), + str_colidx_map); + GPOS_DELETE(str_colidx_map); + } + // serialize properties dxlnode->SerializePropertiesToDXL(xml_serializer); diff --git a/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEProducer.cpp b/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEProducer.cpp index 280fc28b591..84d6155afcb 100644 --- a/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEProducer.cpp +++ b/src/backend/gporca/libnaucrates/src/operators/CDXLPhysicalCTEProducer.cpp @@ -30,8 +30,8 @@ using namespace gpdxl; // //--------------------------------------------------------------------------- CDXLPhysicalCTEProducer::CDXLPhysicalCTEProducer( - CMemoryPool *mp, ULONG id, ULongPtrArray *output_colids_array) - : CDXLPhysical(mp), m_id(id), m_output_colids_array(output_colids_array) + CMemoryPool *mp, ULONG id, ULongPtrArray *output_colids_array, ULongPtrArray *output_colidx_map) + : CDXLPhysical(mp), m_id(id), m_output_colids_array(output_colids_array), m_output_colidx_map(output_colidx_map) { GPOS_ASSERT(nullptr != output_colids_array); } @@ -47,6 +47,7 @@ CDXLPhysicalCTEProducer::CDXLPhysicalCTEProducer( CDXLPhysicalCTEProducer::~CDXLPhysicalCTEProducer() { m_output_colids_array->Release(); + CRefCount::SafeRelease(m_output_colidx_map); } //--------------------------------------------------------------------------- @@ -102,6 +103,15 @@ CDXLPhysicalCTEProducer::SerializeToDXL(CXMLSerializer *xml_serializer, pstrColIds); GPOS_DELETE(pstrColIds); + if (m_output_colidx_map) { + CWStringDynamic *str_colidx_map = + CDXLUtils::Serialize(m_mp, m_output_colidx_map); + + xml_serializer->AddAttribute(CDXLTokens::GetDXLTokenStr(EdxltokenColIdxmap), + str_colidx_map); + GPOS_DELETE(str_colidx_map); + } + // serialize properties dxlnode->SerializePropertiesToDXL(xml_serializer); diff --git a/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEConsumer.cpp b/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEConsumer.cpp index 1ce6036cc8b..69ea4e1f220 100644 --- a/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEConsumer.cpp +++ b/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEConsumer.cpp @@ -74,7 +74,7 @@ CParseHandlerPhysicalCTEConsumer::StartElement( m_dxl_node = GPOS_NEW(m_mp) CDXLNode( m_mp, - GPOS_NEW(m_mp) CDXLPhysicalCTEConsumer(m_mp, id, output_colids_array)); + GPOS_NEW(m_mp) CDXLPhysicalCTEConsumer(m_mp, id, output_colids_array, nullptr)); // parse handler for the proj list CParseHandlerBase *proj_list_parse_handler = diff --git a/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEProducer.cpp b/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEProducer.cpp index 712d7d3324f..4bb3790c08f 100644 --- a/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEProducer.cpp +++ b/src/backend/gporca/libnaucrates/src/parser/CParseHandlerPhysicalCTEProducer.cpp @@ -73,9 +73,14 @@ CParseHandlerPhysicalCTEProducer::StartElement( m_parse_handler_mgr->GetDXLMemoryManager(), attrs, EdxltokenColumns, EdxltokenPhysicalCTEProducer); + ULongPtrArray *output_colidx_map = + CDXLOperatorFactory::ExtractConvertValuesToArray( + m_parse_handler_mgr->GetDXLMemoryManager(), attrs, EdxltokenColIdxmap, + EdxltokenPhysicalCTEProducer); + m_dxl_node = GPOS_NEW(m_mp) CDXLNode( m_mp, - GPOS_NEW(m_mp) CDXLPhysicalCTEProducer(m_mp, id, output_colids_array)); + GPOS_NEW(m_mp) CDXLPhysicalCTEProducer(m_mp, id, output_colids_array, output_colidx_map)); // create and activate the parse handler for the child expression node CParseHandlerBase *child_parse_handler = diff --git a/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp b/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp index 0c977f57529..269319ea440 100644 --- a/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp +++ b/src/backend/gporca/libnaucrates/src/xml/dxltokens.cpp @@ -400,6 +400,7 @@ CDXLTokens::Init(CMemoryPool *mp) {EdxltokenColDescr, GPOS_WSZ_LIT("Column")}, {EdxltokenColRef, GPOS_WSZ_LIT("ColRef")}, + {EdxltokenColIdxmap, GPOS_WSZ_LIT("ColIdxMap")}, {EdxltokenColumns, GPOS_WSZ_LIT("Columns")}, {EdxltokenColumn, GPOS_WSZ_LIT("Column")}, diff --git a/src/backend/gporca/server/src/unittest/gpopt/xforms/CXformTest.cpp b/src/backend/gporca/server/src/unittest/gpopt/xforms/CXformTest.cpp index 1e5f586596a..910a10ad2ab 100644 --- a/src/backend/gporca/server/src/unittest/gpopt/xforms/CXformTest.cpp +++ b/src/backend/gporca/server/src/unittest/gpopt/xforms/CXformTest.cpp @@ -172,6 +172,7 @@ CXformTest::EresUnittest_ApplyXforms_CTE() pdrgpexpr->Append(pexprConsumer); COptCtxt::PoctxtFromTLS()->Pcteinfo()->IncrementConsumers(ulCTEId); + COptCtxt::PoctxtFromTLS()->Pcteinfo()->AddCTEConsumer(pexprNewConsumer); pexprConsumer->AddRef(); CExpression *pexprSelect = diff --git a/src/backend/utils/misc/guc_gp.c b/src/backend/utils/misc/guc_gp.c index a8cbe740829..bd6ae4300da 100644 --- a/src/backend/utils/misc/guc_gp.c +++ b/src/backend/utils/misc/guc_gp.c @@ -311,6 +311,7 @@ bool optimizer_print_group_properties; bool optimizer_print_optimization_context; bool optimizer_print_optimization_stats; bool optimizer_print_xform_results; +bool optimizer_print_preprocess_result; bool optimizer_debug_cte; /* array of xforms disable flags */ @@ -1969,6 +1970,17 @@ struct config_bool ConfigureNamesBool_gp[] = NULL, NULL, NULL }, + { + {"optimizer_print_preprocess_result", PGC_USERSET, LOGGING_WHAT, + gettext_noop("Prints the expression tree produced by the optimizer preprocess(every steps). Only worked with debug version of CBDB."), + NULL, + GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE + }, + &optimizer_print_preprocess_result, + false, + NULL, NULL, NULL + }, + { {"optimizer_print_xform", PGC_USERSET, LOGGING_WHAT, gettext_noop("Prints optimizer transformation information."), @@ -2014,7 +2026,7 @@ struct config_bool ConfigureNamesBool_gp[] = { {"optimizer_debug_cte", PGC_USERSET, LOGGING_WHAT, - gettext_noop("Print the debug info of CTE in ORCA."), + gettext_noop("Print the debug info of CTE in ORCA. Only worked with debug version of CBDB."), NULL, GUC_NO_SHOW_ALL | GUC_NOT_IN_SAMPLE }, diff --git a/src/include/gpopt/translate/CContextDXLToPlStmt.h b/src/include/gpopt/translate/CContextDXLToPlStmt.h index f10d45456f4..29e9eb4401b 100644 --- a/src/include/gpopt/translate/CContextDXLToPlStmt.h +++ b/src/include/gpopt/translate/CContextDXLToPlStmt.h @@ -56,36 +56,31 @@ using HMUlDxltrctx = class CContextDXLToPlStmt { private: - // cte consumer information - struct SCTEConsumerInfo + // cte producer information + struct SCTEEntryInfo { - // list of ShareInputScan represent cte consumers - List *m_cte_consumer_list; + // producer idx mapping + ULongPtrArray *m_pidxmap; - // ctor - SCTEConsumerInfo(List *plan_cte) : m_cte_consumer_list(plan_cte) - { - } + // producer plan + ShareInputScan *m_cte_producer_plan; - void - AddCTEPlan(ShareInputScan *share_input_scan) - { - GPOS_ASSERT(nullptr != share_input_scan); - m_cte_consumer_list = - gpdb::LAppend(m_cte_consumer_list, share_input_scan); - } - ~SCTEConsumerInfo() + // ctor + SCTEEntryInfo(ULongPtrArray *idxmap, ShareInputScan *plan_cte) : + m_pidxmap(idxmap), m_cte_producer_plan(plan_cte) { - gpdb::ListFree(m_cte_consumer_list); + GPOS_ASSERT(plan_cte); } + + ~SCTEEntryInfo() = default; }; - // hash maps mapping ULONG -> SCTEConsumerInfo - using HMUlCTEConsumerInfo = - CHashMap, + // hash maps mapping ULONG -> SCTEEntryInfo + using HMUlCTEProducerInfo = + CHashMap, gpos::Equals, CleanupDelete, - CleanupDelete>; + CleanupDelete>; using HMUlIndex = CHashMap, gpos::Equals, @@ -122,7 +117,7 @@ class CContextDXLToPlStmt ULONG m_result_relation_index; // hash map of the cte identifiers and the cte consumers with the same cte identifier - HMUlCTEConsumerInfo *m_cte_consumer_info; + HMUlCTEProducerInfo *m_cte_producer_info; // CTAS distribution policy GpPolicy *m_distribution_policy; @@ -157,11 +152,11 @@ class CContextDXLToPlStmt // retrieve the next parameter id ULONG GetNextParamId(OID typeoid); - // add a newly found CTE consumer - void AddCTEConsumerInfo(ULONG cte_id, ShareInputScan *share_input_scan); + // register a newly CTE producer + void RegisterCTEProducerInfo(ULONG cte_id, ULongPtrArray *producer_output_colidx_map, ShareInputScan *siscan); - // return the list of shared input scan plans representing the CTE consumers - List *GetCTEConsumerList(ULONG cte_id) const; + // return the shared input scan plans representing the CTE producer + std::pair GetCTEProducerInfo(ULONG cte_id) const; // return list of range table entries List * diff --git a/src/include/utils/guc.h b/src/include/utils/guc.h index 5f7df79ba6e..30a9b3d3122 100644 --- a/src/include/utils/guc.h +++ b/src/include/utils/guc.h @@ -489,6 +489,7 @@ extern bool optimizer_print_group_properties; extern bool optimizer_print_optimization_context; extern bool optimizer_print_optimization_stats; extern bool optimizer_print_xform_results; +extern bool optimizer_print_preprocess_result; extern bool optimizer_debug_cte; /* array of xforms disable flags */ diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index eb0c889c39d..e7be8afc7be 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -475,6 +475,7 @@ "optimizer_print_query", "optimizer_print_xform", "optimizer_print_xform_results", + "optimizer_print_preprocess_result", "optimizer_debug_cte", "optimizer_prune_computed_columns", "optimizer_push_group_by_below_setop_threshold", diff --git a/src/test/regress/expected/cte_prune.out b/src/test/regress/expected/cte_prune.out index 4534dadc4d9..6dd8a640c9b 100644 --- a/src/test/regress/expected/cte_prune.out +++ b/src/test/regress/expected/cte_prune.out @@ -1,32 +1,29 @@ -- start_ignore -drop table if exists t1; -drop table if exists t2; +create schema cte_prune; +set search_path = cte_prune; +SET optimizer_trace_fallback = on; -- end_ignore create table t1(v1 int, v2 int, v3 int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t1 values(generate_series(1, 10), generate_series(11, 20), generate_series(21, 30)); analyze t1; create table t2(v1 int, v2 int, v3 int); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. insert into t2 values(generate_series(0, 100), generate_series(100, 200), generate_series(200, 300)); -- should pruned both seq scan and shared scan explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; - QUERY PLAN ---------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) Output: c11.v1 -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) Output: c11.v1 Hash Cond: (t1.v1 = c11.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.04..1.04 rows=1 width=4) Output: c11.v1 -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=4) Output: c11.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Filter: (t1_1.v1 < 5) Settings: enable_parallel = 'off', optimizer = 'off' @@ -43,20 +40,20 @@ with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 (4 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; - QUERY PLAN ---------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) Output: c11.v2 -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) Output: c11.v2 Hash Cond: (t1.v1 = c11.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.04..1.04 rows=1 width=8) Output: c11.v2, c11.v1 -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=8) Output: c11.v2, c11.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Filter: (t1_1.v1 < 5) Settings: enable_parallel = 'off', optimizer = 'off' @@ -73,20 +70,20 @@ with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 (4 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; - QUERY PLAN ---------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) Output: c11.v3 -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) Output: c11.v3 Hash Cond: (t1.v1 = c11.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.04..1.04 rows=1 width=8) Output: c11.v3, c11.v1 -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=8) Output: c11.v3, c11.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Filter: (t1_1.v1 < 5) Settings: enable_parallel = 'off', optimizer = 'off' @@ -104,20 +101,20 @@ with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 -- * also should be pruned explain verbose with c1 as (select * from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 where c11.v1 < 5; - QUERY PLAN ---------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=1.06..2.16 rows=3 width=4) Output: c11.v1 -> Hash Right Join (cost=1.06..2.11 rows=1 width=4) Output: c11.v1 Hash Cond: (t1.v1 = c11.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.04..1.04 rows=1 width=4) Output: c11.v1 -> Subquery Scan on c11 (cost=0.00..1.04 rows=1 width=4) Output: c11.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.04 rows=1 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Filter: (t1_1.v1 < 5) Settings: enable_parallel = 'off', optimizer = 'off' @@ -147,13 +144,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as Hash Key: c22.v2 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v2 - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=8) Output: c11.v3, c11.v1 -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=8) Output: c11.v3, c11.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -187,13 +184,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as Hash Key: c22.v2 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v2 - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=8) Output: c11.v2, c11.v1 -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=8) Output: c11.v2, c11.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -214,7 +211,7 @@ with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 11 (10 rows) --- distribution col can't pruned +-- distribution col can be pruned which is better than do redistribute in CTE consumer explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; QUERY PLAN ---------------------------------------------------------------------------------------------------- @@ -228,7 +225,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as Hash Key: c11.v2 -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=4) Output: c11.v2 - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.10..1.10 rows=3 width=4) Output: c22.v2 @@ -237,7 +234,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as Hash Key: c22.v2 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v2 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -271,7 +268,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as Hash Key: c11.v3 -> Subquery Scan on c11 (cost=0.00..1.03 rows=3 width=4) Output: c11.v3 - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.10..1.10 rows=3 width=4) Output: c22.v3 @@ -280,7 +277,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as Hash Key: c22.v3 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v3 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -304,8 +301,8 @@ with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 -- groupby/order by/window function/grouping set should be contains in CTE output -- group by explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v1; - QUERY PLAN ---------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=2.17..2.34 rows=10 width=12) Output: (sum(t1.v1)), t1.v1 -> HashAggregate (cost=2.17..2.20 rows=3 width=12) @@ -314,13 +311,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c -> Hash Left Join (cost=1.08..2.15 rows=3 width=4) Output: t1.v1 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -342,8 +339,8 @@ with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left jo (10 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; - QUERY PLAN ---------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=2.24..2.40 rows=10 width=12) Output: (sum(t1.v1)), t1.v2 -> HashAggregate (cost=2.24..2.27 rows=3 width=12) @@ -355,13 +352,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v2, t1.v1 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -383,8 +380,8 @@ with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c1 as c11 left jo (10 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 group by c11.v2; - QUERY PLAN ---------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=2.24..2.40 rows=10 width=12) Output: (sum(t1.v3)), t1.v2 -> HashAggregate (cost=2.24..2.27 rows=3 width=12) @@ -396,13 +393,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v2, t1.v3 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -425,8 +422,8 @@ with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c1 as c11 left jo -- order by explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v1; - QUERY PLAN ---------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) Output: t1.v1, c22.v1 Merge Key: c22.v1 @@ -436,13 +433,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v1, c22.v1 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -464,8 +461,8 @@ with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 (10 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1 order by c22.v3; - QUERY PLAN ---------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=2.18..2.32 rows=10 width=8) Output: t1.v1, c22.v3 Merge Key: c22.v3 @@ -475,13 +472,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v1, c22.v3 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=8) Output: c22.v3, c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=8) Output: c22.v3, c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -504,8 +501,8 @@ with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as c11 left join c1 -- window function explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; - QUERY PLAN ---------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ WindowAgg (cost=2.18..2.47 rows=10 width=12) Output: sum(t1.v1) OVER (?), t1.v2 Order By: t1.v2 @@ -518,13 +515,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER ( -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v2, t1.v1 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -546,8 +543,8 @@ with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER (ORDER BY c11.v2) (10 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; - QUERY PLAN ---------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ WindowAgg (cost=2.18..2.47 rows=10 width=12) Output: sum(t1.v2) OVER (?), t1.v3 Order By: t1.v3 @@ -560,13 +557,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER ( -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v3, t1.v2 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -605,13 +602,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v1, t1.v2 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -660,13 +657,13 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c -> Hash Left Join (cost=1.08..2.15 rows=3 width=8) Output: t1.v2, t1.v3 Hash Cond: (t1.v1 = c22.v1) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: c22.v1 -> Subquery Scan on c22 (cost=0.00..1.03 rows=3 width=4) Output: c22.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v1, t1_1.v2, t1_1.v3 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -700,11 +697,11 @@ select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; -> Hash Join (cost=1.08..362.58 rows=260 width=16) Output: t1.v1, t2.v1, t2.v2, t2.v3 Hash Cond: (t2.v1 = t1.v1) - -> Seq Scan on public.t2 (cost=0.00..293.67 rows=25967 width=12) + -> Seq Scan on cte_prune.t2 (cost=0.00..293.67 rows=25967 width=12) Output: t2.v1, t2.v2, t2.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: t1.v1 - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=4) Output: t1.v1 -> Hash (cost=362.58..362.58 rows=260 width=4) Output: c22.v1 @@ -713,11 +710,11 @@ select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; -> Hash Join (cost=1.08..362.58 rows=260 width=16) Output: t1_1.v1, t2_1.v1, t2_1.v2, t2_1.v3 Hash Cond: (t2_1.v1 = t1_1.v1) - -> Seq Scan on public.t2 t2_1 (cost=0.00..293.67 rows=25967 width=12) + -> Seq Scan on cte_prune.t2 t2_1 (cost=0.00..293.67 rows=25967 width=12) Output: t2_1.v1, t2_1.v2, t2_1.v3 -> Hash (cost=1.03..1.03 rows=3 width=4) Output: t1_1.v1 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=4) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=4) Output: t1_1.v1 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -759,7 +756,7 @@ select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..1.10 rows=3 width=12) Output: t1.v3, t1.v1, t1.v2 Hash Key: t1.v3 - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v3, t1.v1, t1.v2 -> Hash (cost=1.23..1.23 rows=3 width=8) Output: c22.v1 @@ -774,7 +771,7 @@ select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..1.10 rows=3 width=12) Output: t1_1.v3, t1_1.v1, t1_1.v2 Hash Key: t1_1.v3 - -> Seq Scan on public.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 t1_1 (cost=0.00..1.03 rows=3 width=12) Output: t1_1.v3, t1_1.v1, t1_1.v2 Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -786,15 +783,7 @@ select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 fro ----+----+---- (0 rows) --- cte in cte --- function call --- TPCDS cte not support reduce producter output yet --- start_ignore -drop table if exists tpcds_store_sales; -drop table if exists tpcds_date_dim; -drop table if exists tpcds_item; -drop table if exists tpcds_web_sales; --- end_ignore +-- TPCDS case create table tpcds_store_sales ( ss_sold_date_sk integer , @@ -956,7 +945,7 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) -> Hash Join (cost=181.20..223.30 rows=18 width=40) Output: substr((tpcds_item.i_item_desc)::text, 1, 30), tpcds_item.i_item_sk, tpcds_date_dim.d_date Hash Cond: (tpcds_item.i_item_sk = tpcds_store_sales.ss_item_sk) - -> Seq Scan on public.tpcds_item (cost=0.00..39.33 rows=533 width=422) + -> Seq Scan on cte_prune.tpcds_item (cost=0.00..39.33 rows=533 width=422) Output: tpcds_item.i_item_sk, tpcds_item.i_item_id, tpcds_item.i_rec_start_date, tpcds_item.i_rec_end_date, tpcds_item.i_item_desc, tpcds_item.i_current_price, tpcds_item.i_wholesale_cost, tpcds_item.i_brand_id, tpcds_item.i_brand, tpcds_item.i_class_id, tpcds_item.i_class, tpcds_item.i_category_id, tpcds_item.i_category, tpcds_item.i_manufact_id, tpcds_item.i_manufact, tpcds_item.i_size, tpcds_item.i_formulation, tpcds_item.i_color, tpcds_item.i_units, tpcds_item.i_container, tpcds_item.i_manager_id, tpcds_item.i_product_name -> Hash (cost=180.98..180.98 rows=18 width=8) Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date @@ -967,13 +956,13 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date Inner Unique: true Hash Cond: (tpcds_store_sales.ss_sold_date_sk = tpcds_date_dim.d_date_sk) - -> Seq Scan on public.tpcds_store_sales (cost=0.00..79.00 rows=4500 width=8) + -> Seq Scan on cte_prune.tpcds_store_sales (cost=0.00..79.00 rows=4500 width=8) Output: tpcds_store_sales.ss_sold_date_sk, tpcds_store_sales.ss_sold_time_sk, tpcds_store_sales.ss_item_sk, tpcds_store_sales.ss_customer_sk, tpcds_store_sales.ss_cdemo_sk, tpcds_store_sales.ss_hdemo_sk, tpcds_store_sales.ss_addr_sk, tpcds_store_sales.ss_store_sk, tpcds_store_sales.ss_promo_sk, tpcds_store_sales.ss_ticket_number, tpcds_store_sales.ss_quantity, tpcds_store_sales.ss_wholesale_cost, tpcds_store_sales.ss_list_price, tpcds_store_sales.ss_sales_price, tpcds_store_sales.ss_ext_discount_amt, tpcds_store_sales.ss_ext_sales_price, tpcds_store_sales.ss_ext_wholesale_cost, tpcds_store_sales.ss_ext_list_price, tpcds_store_sales.ss_ext_tax, tpcds_store_sales.ss_coupon_amt, tpcds_store_sales.ss_net_paid, tpcds_store_sales.ss_net_paid_inc_tax, tpcds_store_sales.ss_net_profit -> Hash (cost=89.08..89.08 rows=44 width=8) Output: tpcds_date_dim.d_date, tpcds_date_dim.d_date_sk -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..89.08 rows=44 width=8) Output: tpcds_date_dim.d_date, tpcds_date_dim.d_date_sk - -> Seq Scan on public.tpcds_date_dim (cost=0.00..88.50 rows=15 width=8) + -> Seq Scan on cte_prune.tpcds_date_dim (cost=0.00..88.50 rows=15 width=8) Output: tpcds_date_dim.d_date, tpcds_date_dim.d_date_sk Filter: (tpcds_date_dim.d_year = ANY ('{1999,2000,2001,2002}'::integer[])) -> Hash (cost=228.11..228.11 rows=1 width=8) @@ -981,7 +970,7 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) -> Hash Semi Join (cost=227.06..228.11 rows=1 width=8) Output: t1.v1, frequent_ss_items_1.item_sk, RowIdExpr Hash Cond: (t1.v1 = frequent_ss_items_1.item_sk) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=4) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=227.05..227.05 rows=1 width=4) Output: frequent_ss_items_1.item_sk @@ -1005,19 +994,19 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) Output: tpcds_store_sales_1.ss_item_sk, tpcds_date_dim_1.d_date Inner Unique: true Hash Cond: (tpcds_store_sales_1.ss_sold_date_sk = tpcds_date_dim_1.d_date_sk) - -> Seq Scan on public.tpcds_store_sales tpcds_store_sales_1 (cost=0.00..90.25 rows=1500 width=8) + -> Seq Scan on cte_prune.tpcds_store_sales tpcds_store_sales_1 (cost=0.00..90.25 rows=1500 width=8) Output: tpcds_store_sales_1.ss_sold_date_sk, tpcds_store_sales_1.ss_sold_time_sk, tpcds_store_sales_1.ss_item_sk, tpcds_store_sales_1.ss_customer_sk, tpcds_store_sales_1.ss_cdemo_sk, tpcds_store_sales_1.ss_hdemo_sk, tpcds_store_sales_1.ss_addr_sk, tpcds_store_sales_1.ss_store_sk, tpcds_store_sales_1.ss_promo_sk, tpcds_store_sales_1.ss_ticket_number, tpcds_store_sales_1.ss_quantity, tpcds_store_sales_1.ss_wholesale_cost, tpcds_store_sales_1.ss_list_price, tpcds_store_sales_1.ss_sales_price, tpcds_store_sales_1.ss_ext_discount_amt, tpcds_store_sales_1.ss_ext_sales_price, tpcds_store_sales_1.ss_ext_wholesale_cost, tpcds_store_sales_1.ss_ext_list_price, tpcds_store_sales_1.ss_ext_tax, tpcds_store_sales_1.ss_coupon_amt, tpcds_store_sales_1.ss_net_paid, tpcds_store_sales_1.ss_net_paid_inc_tax, tpcds_store_sales_1.ss_net_profit Filter: (tpcds_store_sales_1.ss_item_sk > 0) -> Hash (cost=89.08..89.08 rows=44 width=8) Output: tpcds_date_dim_1.d_date, tpcds_date_dim_1.d_date_sk -> Broadcast Motion 3:3 (slice6; segments: 3) (cost=0.00..89.08 rows=44 width=8) Output: tpcds_date_dim_1.d_date, tpcds_date_dim_1.d_date_sk - -> Seq Scan on public.tpcds_date_dim tpcds_date_dim_1 (cost=0.00..88.50 rows=15 width=8) + -> Seq Scan on cte_prune.tpcds_date_dim tpcds_date_dim_1 (cost=0.00..88.50 rows=15 width=8) Output: tpcds_date_dim_1.d_date, tpcds_date_dim_1.d_date_sk Filter: (tpcds_date_dim_1.d_year = ANY ('{1999,2000,2001,2002}'::integer[])) -> Hash (cost=40.67..40.67 rows=178 width=422) Output: tpcds_item_1.i_item_desc, tpcds_item_1.i_item_sk - -> Seq Scan on public.tpcds_item tpcds_item_1 (cost=0.00..40.67 rows=178 width=422) + -> Seq Scan on cte_prune.tpcds_item tpcds_item_1 (cost=0.00..40.67 rows=178 width=422) Output: tpcds_item_1.i_item_desc, tpcds_item_1.i_item_sk Filter: (tpcds_item_1.i_item_sk > 0) Settings: enable_parallel = 'off', optimizer = 'off' @@ -1031,8 +1020,8 @@ explain verbose with ws_wh as where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=2952.11..2952.33 rows=14 width=12) Output: t1.v1, t1.v2, t1.v3 -> HashAggregate (cost=2952.11..2952.15 rows=5 width=12) @@ -1053,14 +1042,14 @@ select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..137.00 rows=3433 width=8) Output: ws1.ws_order_number, ws1.ws_warehouse_sk Hash Key: ws1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..68.33 rows=3433 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws1 (cost=0.00..68.33 rows=3433 width=8) Output: ws1.ws_order_number, ws1.ws_warehouse_sk -> Hash (cost=137.00..137.00 rows=3433 width=8) Output: ws2.ws_warehouse_sk, ws2.ws_order_number -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..137.00 rows=3433 width=8) Output: ws2.ws_warehouse_sk, ws2.ws_order_number Hash Key: ws2.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws2 (cost=0.00..68.33 rows=3433 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws2 (cost=0.00..68.33 rows=3433 width=8) Output: ws2.ws_warehouse_sk, ws2.ws_order_number -> Hash (cost=328.14..328.14 rows=5 width=16) Output: t1.v1, t1.v2, t1.v3, ws_wh_1.ws_order_number, (RowIdExpr) @@ -1087,7 +1076,7 @@ select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..99.81 rows=1144 width=8) Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk Hash Key: ws1_1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws1_1 (cost=0.00..76.92 rows=1144 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws1_1 (cost=0.00..76.92 rows=1144 width=8) Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk Filter: (ws1_1.ws_order_number > 0) -> Hash (cost=99.81..99.81 rows=1144 width=8) @@ -1095,12 +1084,12 @@ select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in -> Redistribute Motion 3:3 (slice8; segments: 3) (cost=0.00..99.81 rows=1144 width=8) Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number Hash Key: ws2_1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws2_1 (cost=0.00..76.92 rows=1144 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws2_1 (cost=0.00..76.92 rows=1144 width=8) Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number Filter: (ws2_1.ws_order_number > 0) -> Hash (cost=1.03..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3, RowIdExpr Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -1112,8 +1101,8 @@ explain verbose with ws_wh as where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); - QUERY PLAN --------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=2952.68..2952.91 rows=14 width=12) Output: t1.v1, t1.v2, t1.v3 -> HashAggregate (cost=2952.68..2952.73 rows=5 width=12) @@ -1134,14 +1123,14 @@ select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..137.00 rows=3433 width=8) Output: ws1.ws_order_number, ws1.ws_warehouse_sk Hash Key: ws1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..68.33 rows=3433 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws1 (cost=0.00..68.33 rows=3433 width=8) Output: ws1.ws_order_number, ws1.ws_warehouse_sk -> Hash (cost=137.00..137.00 rows=3433 width=8) Output: ws2.ws_warehouse_sk, ws2.ws_order_number -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..137.00 rows=3433 width=8) Output: ws2.ws_warehouse_sk, ws2.ws_order_number Hash Key: ws2.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws2 (cost=0.00..68.33 rows=3433 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws2 (cost=0.00..68.33 rows=3433 width=8) Output: ws2.ws_warehouse_sk, ws2.ws_order_number -> Hash (cost=328.44..328.44 rows=14 width=16) Output: t1.v1, t1.v2, t1.v3, ws_wh_1.wh1, (RowIdExpr) @@ -1167,7 +1156,7 @@ select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..99.81 rows=1144 width=8) Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk Hash Key: ws1_1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws1_1 (cost=0.00..76.92 rows=1144 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws1_1 (cost=0.00..76.92 rows=1144 width=8) Output: ws1_1.ws_order_number, ws1_1.ws_warehouse_sk Filter: (ws1_1.ws_order_number > 0) -> Hash (cost=99.81..99.81 rows=1144 width=8) @@ -1175,14 +1164,14 @@ select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 -> Redistribute Motion 3:3 (slice8; segments: 3) (cost=0.00..99.81 rows=1144 width=8) Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number Hash Key: ws2_1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws2_1 (cost=0.00..76.92 rows=1144 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws2_1 (cost=0.00..76.92 rows=1144 width=8) Output: ws2_1.ws_warehouse_sk, ws2_1.ws_order_number Filter: (ws2_1.ws_order_number > 0) -> Hash (cost=1.17..1.17 rows=10 width=12) Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) -> Broadcast Motion 3:3 (slice9; segments: 3) (cost=0.00..1.17 rows=10 width=12) Output: t1.v1, t1.v2, t1.v3, (RowIdExpr) - -> Seq Scan on public.t1 (cost=0.00..1.03 rows=3 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..1.03 rows=3 width=12) Output: t1.v1, t1.v2, t1.v3, RowIdExpr Settings: enable_parallel = 'off', optimizer = 'off' Optimizer: Postgres query optimizer @@ -1196,3 +1185,591 @@ drop table tpcds_web_sales; drop table t1; drop table t2; -- end_ignore +-- comm cases +CREATE TABLE t3 AS SELECT i as a, i+1 as b from generate_series(1,10)i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE t4 AS SELECT i as c, i+1 as d from generate_series(1,10)i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'c' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- Additional filtering conditions are added to the consumer. +-- This is caused by `PexprInferPredicates` in the ORCA preprocessor. +explain verbose WITH t(a,b,d) AS +( + SELECT t3.a,t3.b,t4.d FROM t3,t4 WHERE t3.a = t4.d +) +SELECT cup.*, SUM(t.d) OVER(PARTITION BY t.b) FROM + ( + SELECT t4.*, AVG(t.b) OVER(PARTITION BY t.a ORDER BY t.b desc) AS e FROM t,t4 + ) AS cup, +t WHERE cup.e < 10 +GROUP BY cup.c,cup.d, cup.e ,t.d, t.b +ORDER BY 1,2,3,4 +LIMIT 10; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=102679372549802016.00..102679372549802016.00 rows=10 width=56) + Output: cup.c, cup.d, cup.e, (sum(t.d) OVER (?)), t.d, t.b + -> Gather Motion 3:1 (slice1; segments: 3) (cost=102679372549802016.00..102679372549802016.00 rows=30 width=56) + Output: cup.c, cup.d, cup.e, (sum(t.d) OVER (?)), t.d, t.b + Merge Key: cup.c, cup.d, cup.e, (sum(t.d) OVER (?)) + -> Limit (cost=102679372549802016.00..102679372549802016.00 rows=10 width=56) + Output: cup.c, cup.d, cup.e, (sum(t.d) OVER (?)), t.d, t.b + -> Sort (cost=102679372549802016.00..102679990317302016.00 rows=247107000000000 width=56) + Output: cup.c, cup.d, cup.e, (sum(t.d) OVER (?)), t.d, t.b + Sort Key: cup.c, cup.d, cup.e, (sum(t.d) OVER (?)) + -> WindowAgg (cost=102669708283873296.00..102674032656373296.00 rows=247107000000000 width=56) + Output: cup.c, cup.d, cup.e, sum(t.d) OVER (?), t.d, t.b + Partition By: t.b + -> Sort (cost=102669708283873296.00..102670326051373296.00 rows=247107000000000 width=48) + Output: cup.c, cup.d, cup.e, t.d, t.b + Sort Key: t.b + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=102580688686565152.00..102601132929768272.00 rows=247107000000000 width=48) + Output: cup.c, cup.d, cup.e, t.d, t.b + Hash Key: t.b + -> HashAggregate (cost=102580688686565152.00..102596190789768272.00 rows=247107000000000 width=48) + Output: cup.c, cup.d, cup.e, t.d, t.b + Group Key: cup.c, cup.d, cup.e, t.d, t.b + Planned Partitions: 256 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=93196181903903024.00..102459992361252656.00 rows=741321000000000 width=48) + Output: cup.c, cup.d, cup.e, t.d, t.b + Hash Key: cup.c, cup.d, cup.e, t.d, t.b + -> HashAggregate (cost=93196181903903024.00..102445165941252656.00 rows=741321000000000 width=48) + Output: cup.c, cup.d, cup.e, t.d, t.b + Group Key: cup.c, cup.d, cup.e, t.d, t.b + Planned Partitions: 256 + -> Nested Loop (cost=67221234716.02..7598699218584692.00 rows=525742695955889984 width=48) + Output: cup.c, cup.d, cup.e, t.d, t.b + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=679.75..228161.25 rows=7413210 width=8) + Output: t.d, t.b + -> Subquery Scan on t (cost=679.75..129318.45 rows=2471070 width=8) + Output: t.d, t.b + -> Hash Join (cost=679.75..129318.45 rows=2471070 width=12) + Output: t3.a, t3.b, t4.d + Hash Cond: (t4.d = t3.a) + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..895.00 rows=28700 width=4) + Output: t4.d + Hash Key: t4.d + -> Seq Scan on cte_prune.t4 (cost=0.00..321.00 rows=28700 width=4) + Output: t4.d + -> Hash (cost=321.00..321.00 rows=28700 width=8) + Output: t3.a, t3.b + -> Seq Scan on cte_prune.t3 (cost=0.00..321.00 rows=28700 width=8) + Output: t3.a, t3.b + -> Materialize (cost=57221234036.27..64629019265.77 rows=70919709000 width=40) + Output: cup.c, cup.d, cup.e + -> Subquery Scan on cup (cost=57221234036.27..64135905663.77 rows=70919709000 width=40) + Output: cup.c, cup.d, cup.e + Filter: (cup.e < '10'::numeric) + -> WindowAgg (cost=57221234036.27..61476416576.27 rows=212759127000 width=48) + Output: t4_1.c, t4_1.d, avg(t3_1.b) OVER (?), t3_1.b, t3_1.a + Partition By: t3_1.a + Order By: t3_1.b + -> Sort (cost=57221234036.27..57753131853.77 rows=212759127000 width=16) + Output: t3_1.b, t3_1.a, t4_1.c, t4_1.d + Sort Key: t3_1.a, t3_1.b DESC + -> Nested Loop (cost=10000000679.75..14463131602.75 rows=212759127000 width=16) + Output: t3_1.b, t3_1.a, t4_1.c, t4_1.d + -> Broadcast Motion 3:3 (slice6; segments: 3) (cost=0.00..1469.00 rows=86100 width=8) + Output: t4_1.c, t4_1.d + -> Seq Scan on cte_prune.t4 t4_1 (cost=0.00..321.00 rows=28700 width=8) + Output: t4_1.c, t4_1.d + -> Materialize (cost=679.75..141673.80 rows=2471070 width=8) + Output: t3_1.a, t3_1.b, t4_2.d + -> Hash Join (cost=679.75..129318.45 rows=2471070 width=12) + Output: t3_1.a, t3_1.b, t4_2.d + Hash Cond: (t4_2.d = t3_1.a) + -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..895.00 rows=28700 width=4) + Output: t4_2.d + Hash Key: t4_2.d + -> Seq Scan on cte_prune.t4 t4_2 (cost=0.00..321.00 rows=28700 width=4) + Output: t4_2.d + -> Hash (cost=321.00..321.00 rows=28700 width=8) + Output: t3_1.a, t3_1.b + -> Seq Scan on cte_prune.t3 t3_1 (cost=0.00..321.00 rows=28700 width=8) + Output: t3_1.a, t3_1.b + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(82 rows) + +WITH t(a,b,d) AS +( + SELECT t3.a,t3.b,t4.d FROM t3,t4 WHERE t3.a = t4.d +) +SELECT cup.*, SUM(t.d) OVER(PARTITION BY t.b) FROM + ( + SELECT t4.*, AVG(t.b) OVER(PARTITION BY t.a ORDER BY t.b desc) AS e FROM t,t4 + ) AS cup, +t WHERE cup.e < 10 +GROUP BY cup.c,cup.d, cup.e ,t.d, t.b +ORDER BY 1,2,3,4 +LIMIT 10; + c | d | e | sum +---+---+--------------------+----- + 1 | 2 | 3.0000000000000000 | 140 + 1 | 2 | 3.0000000000000000 | 210 + 1 | 2 | 3.0000000000000000 | 280 + 1 | 2 | 3.0000000000000000 | 350 + 1 | 2 | 3.0000000000000000 | 420 + 1 | 2 | 3.0000000000000000 | 490 + 1 | 2 | 3.0000000000000000 | 560 + 1 | 2 | 3.0000000000000000 | 630 + 1 | 2 | 3.0000000000000000 | 700 + 1 | 2 | 4.0000000000000000 | 140 +(10 rows) + +-- grouping set will generate the internal CTE. +CREATE TABLE cte_prune_tenk1 (unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'unique1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO cte_prune_tenk1 +SELECT + i AS unique1, + (i + 10000) AS unique2, + i % 2 AS two, + i % 4 AS four, + i % 10 AS ten, + i % 20 AS twenty, + i % 100 AS hundred, + i % 1000 AS thousand, + i % 2000 AS twothousand, + i % 5000 AS fivethous, + i % 10000 AS tenthous, + (2 * i + 1) AS odd, + (2 * i) AS even, + ('A' || lpad(i::text, 4, '0'))::name AS stringu1, + ('B' || lpad(i::text, 4, '0'))::name AS stringu2, + (CASE (i % 4) + WHEN 0 THEN 'AAAA'::name + WHEN 1 THEN 'BBBB'::name + WHEN 2 THEN 'CCCC'::name + ELSE 'DDDD'::name + END) AS string4 +FROM generate_series(0, 99) AS i; +explain verbose select four, x + from (select four, ten, 'foo'::text as x from cte_prune_tenk1) as t + group by grouping sets (four, x) + having x = 'foo'; + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=130.74..134.95 rows=3 width=36) + Output: cte_prune_tenk1.four, ('foo'::text) + -> HashAggregate (cost=130.74..134.91 rows=1 width=36) + Output: cte_prune_tenk1.four, ('foo'::text) + Group Key: cte_prune_tenk1.four, ('foo'::text), (GROUPINGSET_ID()) + Filter: (('foo'::text) = 'foo'::text) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=83.58..123.29 rows=993 width=36) + Output: cte_prune_tenk1.four, ('foo'::text), (GROUPINGSET_ID()) + Hash Key: cte_prune_tenk1.four, ('foo'::text), (GROUPINGSET_ID()) + -> Partial HashAggregate (cost=83.58..103.43 rows=993 width=36) + Output: cte_prune_tenk1.four, ('foo'::text), GROUPINGSET_ID() + Hash Key: cte_prune_tenk1.four + Hash Key: 'foo'::text + -> Seq Scan on cte_prune.cte_prune_tenk1 (cost=0.00..73.67 rows=3967 width=36) + Output: cte_prune_tenk1.four, 'foo'::text + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(17 rows) + +select four, x + from (select four, ten, 'foo'::text as x from cte_prune_tenk1) as t + group by grouping sets (four, x) + having x = 'foo'; + four | x +------+----- + | foo +(1 row) + +-- nest CTE cases +-- start_ignore +drop table city; +ERROR: table "city" does not exist +drop table country; +ERROR: table "country" does not exist +drop table countrylanguage; +ERROR: table "countrylanguage" does not exist +-- end_ignore +CREATE TABLE city ( + id integer NOT NULL, + name text NOT NULL, + countrycode character(3) NOT NULL, + district text NOT NULL, + population integer NOT NULL +) distributed by(id); +CREATE TABLE country ( + code character(3) NOT NULL, + name text NOT NULL, + continent text NOT NULL, + region text NOT NULL, + surfacearea numeric(10,2) NOT NULL, + indepyear smallint, + population integer NOT NULL, + lifeexpectancy real, + gnp numeric(10,2), + gnpold numeric(10,2), + localname text NOT NULL, + governmentform text NOT NULL, + headofstate text, + capital integer, + code2 character(2) NOT NULL +) distributed by (code); +CREATE TABLE countrylanguage ( + countrycode character(3) NOT NULL, + "language" text NOT NULL, + isofficial boolean NOT NULL, + percentage real NOT NULL +)distributed by (countrycode,language); +ALTER TABLE ONLY city + ADD CONSTRAINT city_pkey PRIMARY KEY (id); +ALTER TABLE ONLY country + ADD CONSTRAINT country_pkey PRIMARY KEY (code); +ALTER TABLE ONLY countrylanguage + ADD CONSTRAINT countrylanguage_pkey PRIMARY KEY (countrycode, "language"); +-- CTE1(inlined) in CTE2(no-inlined) case +explain verbose with country as +(select country.code,country.name COUNTRY, city.name CAPITAL, language, isofficial, percentage + FROM country,city,countrylanguage + WHERE country.code = countrylanguage.countrycode + and country.capital = city.id + and country.continent = 'Europe'), +countrylanguage as +(select country.code,country.COUNTRY,country.language,country.isofficial,country.percentage + FROM country,countrylanguage + WHERE country.code = countrylanguage.countrycode +) +select * from +(select * from country where isofficial='True') country, +(select * from countrylanguage where percentage > 50) countrylanguage +where country.percentage = countrylanguage.percentage order by countrylanguage.COUNTRY,country.language LIMIT 40; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=1051.23..1051.63 rows=28 width=202) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage, country.code, country.country, country.language, country.isofficial, country.percentage + -> Gather Motion 3:1 (slice1; segments: 3) (cost=1051.23..1051.63 rows=28 width=202) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage, country.code, country.country, country.language, country.isofficial, country.percentage + Merge Key: country.country, countrylanguage_2.language + -> Limit (cost=1051.23..1051.26 rows=9 width=202) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage, country.code, country.country, country.language, country.isofficial, country.percentage + -> Sort (cost=1051.23..1051.26 rows=9 width=202) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage, country.code, country.country, country.language, country.isofficial, country.percentage + Sort Key: country.country, countrylanguage_2.language + -> Hash Join (cost=846.99..1051.08 rows=9 width=202) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage, country.code, country.country, country.language, country.isofficial, country.percentage + Hash Cond: (country.percentage = countrylanguage_2.percentage) + -> Hash Join (cost=436.98..637.91 rows=245 width=85) + Output: country.code, country.country, country.language, country.isofficial, country.percentage + Hash Cond: (countrylanguage.countrycode = country.code) + -> Seq Scan on cte_prune.countrylanguage (cost=0.00..165.33 rows=13133 width=16) + Output: countrylanguage.countrycode, countrylanguage.language, countrylanguage.isofficial, countrylanguage.percentage + -> Hash (cost=436.74..436.74 rows=19 width=85) + Output: country.code, country.country, country.language, country.isofficial, country.percentage + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=227.29..436.74 rows=19 width=85) + Output: country.code, country.country, country.language, country.isofficial, country.percentage + -> Subquery Scan on country (cost=227.29..436.49 rows=6 width=85) + Output: country.code, country.country, country.language, country.isofficial, country.percentage + -> Hash Join (cost=227.29..436.49 rows=6 width=117) + Output: country_1.code, country_1.name, city.name, countrylanguage_1.language, countrylanguage_1.isofficial, countrylanguage_1.percentage + Hash Cond: (countrylanguage_1.countrycode = country_1.code) + -> Seq Scan on cte_prune.countrylanguage countrylanguage_1 (cost=0.00..198.17 rows=4378 width=53) + Output: countrylanguage_1.countrycode, countrylanguage_1.language, countrylanguage_1.isofficial, countrylanguage_1.percentage + Filter: (countrylanguage_1.percentage > '50'::double precision) + -> Hash (cost=227.11..227.11 rows=15 width=80) + Output: country_1.code, country_1.name, city.name + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=77.45..227.11 rows=15 width=80) + Output: country_1.code, country_1.name, city.name + -> Hash Join (cost=77.45..226.91 rows=5 width=80) + Output: country_1.code, country_1.name, city.name + Hash Cond: (city.id = country_1.capital) + -> Seq Scan on cte_prune.city (cost=0.00..126.33 rows=9233 width=36) + Output: city.id, city.name, city.countrycode, city.district, city.population + -> Hash (cost=77.40..77.40 rows=3 width=52) + Output: country_1.code, country_1.name, country_1.capital + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..77.40 rows=3 width=52) + Output: country_1.code, country_1.name, country_1.capital + Hash Key: country_1.capital + -> Seq Scan on cte_prune.country country_1 (cost=0.00..77.33 rows=3 width=52) + Output: country_1.code, country_1.name, country_1.capital + Filter: (country_1.continent = 'Europe'::text) + -> Hash (cost=409.67..409.67 rows=28 width=117) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage + -> Broadcast Motion 3:3 (slice5; segments: 3) (cost=227.29..409.67 rows=28 width=117) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage + -> Hash Join (cost=227.29..409.20 rows=9 width=117) + Output: country_2.code, country_2.name, city_1.name, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage + Hash Cond: (countrylanguage_2.countrycode = country_2.code) + -> Seq Scan on cte_prune.countrylanguage countrylanguage_2 (cost=0.00..165.33 rows=6567 width=53) + Output: countrylanguage_2.countrycode, countrylanguage_2.language, countrylanguage_2.isofficial, countrylanguage_2.percentage + Filter: countrylanguage_2.isofficial + -> Hash (cost=227.11..227.11 rows=15 width=80) + Output: country_2.code, country_2.name, city_1.name + -> Broadcast Motion 3:3 (slice6; segments: 3) (cost=77.45..227.11 rows=15 width=80) + Output: country_2.code, country_2.name, city_1.name + -> Hash Join (cost=77.45..226.91 rows=5 width=80) + Output: country_2.code, country_2.name, city_1.name + Hash Cond: (city_1.id = country_2.capital) + -> Seq Scan on cte_prune.city city_1 (cost=0.00..126.33 rows=9233 width=36) + Output: city_1.id, city_1.name, city_1.countrycode, city_1.district, city_1.population + -> Hash (cost=77.40..77.40 rows=3 width=52) + Output: country_2.code, country_2.name, country_2.capital + -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..77.40 rows=3 width=52) + Output: country_2.code, country_2.name, country_2.capital + Hash Key: country_2.capital + -> Seq Scan on cte_prune.country country_2 (cost=0.00..77.33 rows=3 width=52) + Output: country_2.code, country_2.name, country_2.capital + Filter: (country_2.continent = 'Europe'::text) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(76 rows) + +-- CTE in the main query and subqueries within the main query +explain verbose with bad_headofstates as +( + select country.code,country.name,country.headofstate,countrylanguage.language + from + country,countrylanguage + where country.code = countrylanguage.countrycode and countrylanguage.isofficial=true + and (country.gnp < country.gnpold or country.gnp < 3000) +) +select OUTERMOST_FOO.*,bad_headofstates.headofstate from ( +select avg(population),region from +( +select FOO.*,bad_headofstates.headofstate,city.name +from +(select bad_headofstates.code,country.capital,country.region,country.population from +bad_headofstates,country where bad_headofstates.code = country.code) FOO, bad_headofstates,city +where FOO.code = bad_headofstates.code and FOO.capital = city.id) OUTER_FOO +group by region ) OUTERMOST_FOO,bad_headofstates,country +where country.code = bad_headofstates.code and country.region = OUTERMOST_FOO.region +order by OUTERMOST_FOO.region,bad_headofstates.headofstate LIMIT 40; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Limit (cost=3473.37..3473.94 rows=40 width=96) + Output: (avg(country_2.population)), country_2.region, country_1.headofstate + -> Gather Motion 3:1 (slice1; segments: 3) (cost=3473.37..3475.07 rows=120 width=96) + Output: (avg(country_2.population)), country_2.region, country_1.headofstate + Merge Key: country_2.region, country_1.headofstate + -> Limit (cost=3473.37..3473.47 rows=40 width=96) + Output: (avg(country_2.population)), country_2.region, country_1.headofstate + -> Sort (cost=3473.37..3482.49 rows=3648 width=96) + Output: (avg(country_2.population)), country_2.region, country_1.headofstate + Sort Key: country_2.region, country_1.headofstate + -> Hash Join (cost=2915.55..3358.06 rows=3648 width=96) + Output: (avg(country_2.population)), country_2.region, country_1.headofstate + Inner Unique: true + Hash Cond: (country.region = country_2.region) + -> Hash Join (cost=222.07..618.52 rows=3648 width=64) + Output: country_1.headofstate, country.region + Inner Unique: true + Hash Cond: (country_1.code = country.code) + -> Hash Join (cost=110.07..460.46 rows=3648 width=112) + Output: country_1.code, country_1.name, country_1.headofstate, countrylanguage.language + Inner Unique: true + Hash Cond: (countrylanguage.countrycode = country_1.code) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..296.67 rows=6567 width=48) + Output: countrylanguage.language, countrylanguage.countrycode + Hash Key: countrylanguage.countrycode + -> Seq Scan on cte_prune.countrylanguage (cost=0.00..165.33 rows=6567 width=48) + Output: countrylanguage.language, countrylanguage.countrycode + Filter: countrylanguage.isofficial + -> Hash (cost=86.00..86.00 rows=1926 width=80) + Output: country_1.code, country_1.name, country_1.headofstate + -> Seq Scan on cte_prune.country country_1 (cost=0.00..86.00 rows=1926 width=80) + Output: country_1.code, country_1.name, country_1.headofstate + Filter: ((country_1.gnp < country_1.gnpold) OR (country_1.gnp < '3000'::numeric)) + -> Hash (cost=68.67..68.67 rows=3467 width=48) + Output: country.code, country.region + -> Seq Scan on cte_prune.country (cost=0.00..68.67 rows=3467 width=48) + Output: country.code, country.region + -> Hash (cost=2680.97..2680.97 rows=1000 width=64) + Output: (avg(country_2.population)), country_2.region + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=2660.14..2680.97 rows=1000 width=64) + Output: (avg(country_2.population)), country_2.region + -> Finalize HashAggregate (cost=2660.14..2664.31 rows=333 width=64) + Output: avg(country_2.population), country_2.region + Group Key: country_2.region + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=2625.14..2655.14 rows=1000 width=64) + Output: country_2.region, (PARTIAL avg(country_2.population)) + Hash Key: country_2.region + -> Partial HashAggregate (cost=2625.14..2635.14 rows=1000 width=64) + Output: country_2.region, PARTIAL avg(country_2.population) + Group Key: country_2.region + -> Hash Join (cost=1662.17..2425.51 rows=39927 width=36) + Output: country_2.region, country_2.population + Hash Cond: (country_3.code = country_2.code) + -> Hash Join (cost=110.07..460.46 rows=3648 width=112) + Output: country_3.code, country_3.name, country_3.headofstate, countrylanguage_1.language + Inner Unique: true + Hash Cond: (countrylanguage_1.countrycode = country_3.code) + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..296.67 rows=6567 width=48) + Output: countrylanguage_1.language, countrylanguage_1.countrycode + Hash Key: countrylanguage_1.countrycode + -> Seq Scan on cte_prune.countrylanguage countrylanguage_1 (cost=0.00..165.33 rows=6567 width=48) + Output: countrylanguage_1.language, countrylanguage_1.countrycode + Filter: countrylanguage_1.isofficial + -> Hash (cost=86.00..86.00 rows=1926 width=80) + Output: country_3.code, country_3.name, country_3.headofstate + -> Seq Scan on cte_prune.country country_3 (cost=0.00..86.00 rows=1926 width=80) + Output: country_3.code, country_3.name, country_3.headofstate + Filter: ((country_3.gnp < country_3.gnpold) OR (country_3.gnp < '3000'::numeric)) + -> Hash (cost=1506.49..1506.49 rows=3648 width=68) + Output: country_2.population, country_2.region, country_2.code, country_4.code + -> Hash Join (cost=1063.99..1506.49 rows=3648 width=68) + Output: country_2.population, country_2.region, country_2.code, country_4.code + Inner Unique: true + Hash Cond: (country_2.capital = city.id) + -> Hash Join (cost=222.07..618.52 rows=3648 width=72) + Output: country_2.population, country_2.region, country_2.code, country_2.capital, country_4.code + Inner Unique: true + Hash Cond: (country_4.code = country_2.code) + -> Hash Join (cost=110.07..460.46 rows=3648 width=112) + Output: country_4.code, country_4.name, country_4.headofstate, countrylanguage_2.language + Inner Unique: true + Hash Cond: (countrylanguage_2.countrycode = country_4.code) + -> Redistribute Motion 3:3 (slice6; segments: 3) (cost=0.00..296.67 rows=6567 width=48) + Output: countrylanguage_2.language, countrylanguage_2.countrycode + Hash Key: countrylanguage_2.countrycode + -> Seq Scan on cte_prune.countrylanguage countrylanguage_2 (cost=0.00..165.33 rows=6567 width=48) + Output: countrylanguage_2.language, countrylanguage_2.countrycode + Filter: countrylanguage_2.isofficial + -> Hash (cost=86.00..86.00 rows=1926 width=80) + Output: country_4.code, country_4.name, country_4.headofstate + -> Seq Scan on cte_prune.country country_4 (cost=0.00..86.00 rows=1926 width=80) + Output: country_4.code, country_4.name, country_4.headofstate + Filter: ((country_4.gnp < country_4.gnpold) OR (country_4.gnp < '3000'::numeric)) + -> Hash (cost=68.67..68.67 rows=3467 width=56) + Output: country_2.population, country_2.region, country_2.code, country_2.capital + -> Seq Scan on cte_prune.country country_2 (cost=0.00..68.67 rows=3467 width=56) + Output: country_2.population, country_2.region, country_2.code, country_2.capital + -> Hash (cost=495.67..495.67 rows=27700 width=4) + Output: city.id + -> Broadcast Motion 3:3 (slice7; segments: 3) (cost=0.00..495.67 rows=27700 width=4) + Output: city.id + -> Seq Scan on cte_prune.city (cost=0.00..126.33 rows=9233 width=4) + Output: city.id + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(105 rows) + +-- start_ignore +drop table city; +drop table country; +drop table countrylanguage; +-- end_ignore +-- inlined CTEs +CREATE TABLE t5 AS SELECT i as c, i+1 as d from generate_series(1,10)i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'c' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE t6 AS SELECT i as a, i+1 as b from generate_series(1,10)i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- inlined CTEs should have not unused columns(ex. t5.*, t6.* in output) +explain verbose WITH w AS (SELECT a, b from t6 where b < 5) +SELECT * +FROM t6, + (WITH v AS (SELECT c, d FROM t5, w WHERE c = w.a AND c < 2) + SELECT v1.c, v1.d FROM v v1, v v2 WHERE v1.c = v2.c AND v1.d > 1 + ) x +WHERE t6.a = x.c ORDER BY 1; + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=824331921.87..983595523.10 rows=11242136557 width=16) + Output: t6.a, t6.b, t5_1.c, t5_1.d + Merge Key: t6.a + -> Sort (cost=824331921.87..833700369.00 rows=3747378852 width=16) + Output: t6.a, t6.b, t5_1.c, t5_1.d + Sort Key: t6.a + -> Hash Join (cost=5001.56..196417010.27 rows=3747378852 width=16) + Output: t6.a, t6.b, t5_1.c, t5_1.d + Hash Cond: (t5_1.c = t6.a) + -> Hash Join (cost=4321.81..2257711.94 rows=43523564 width=8) + Output: t5_1.c, t5_1.d + Hash Cond: (t5.c = t5_1.c) + -> Hash Join (cost=512.33..4788.69 rows=158519 width=8) + Output: t5.c, t5.d + Hash Cond: (t6_1.a = t5.c) + -> Seq Scan on cte_prune.t6 t6_1 (cost=0.00..464.50 rows=5523 width=8) + Output: t6_1.a, t6_1.b + Filter: ((t6_1.b < 5) AND (t6_1.a < 2)) + -> Hash (cost=392.75..392.75 rows=9567 width=8) + Output: t5.c, t5.d + -> Seq Scan on cte_prune.t5 (cost=0.00..392.75 rows=9567 width=8) + Output: t5.c, t5.d + Filter: (t5.c < 2) + -> Hash (cost=2665.47..2665.47 rows=91521 width=8) + Output: t5_1.c, t5_1.d + -> Hash Join (cost=533.54..2665.47 rows=91521 width=8) + Output: t5_1.c, t5_1.d + Hash Cond: (t5_1.c = w.a) + -> Seq Scan on cte_prune.t5 t5_1 (cost=0.00..464.50 rows=5523 width=8) + Output: t5_1.c, t5_1.d + Filter: ((t5_1.c < 2) AND (t5_1.d > 1)) + -> Hash (cost=464.50..464.50 rows=5523 width=4) + Output: w.a + -> Subquery Scan on w (cost=0.00..464.50 rows=5523 width=4) + Output: w.a + -> Seq Scan on cte_prune.t6 t6_2 (cost=0.00..464.50 rows=5523 width=8) + Output: t6_2.a, t6_2.b + Filter: ((t6_2.b < 5) AND (t6_2.a < 2)) + -> Hash (cost=321.00..321.00 rows=28700 width=8) + Output: t6.a, t6.b + -> Seq Scan on cte_prune.t6 (cost=0.00..321.00 rows=28700 width=8) + Output: t6.a, t6.b + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(44 rows) + +WITH w AS (SELECT a, b from t6 where b < 5) +SELECT * +FROM t6, + (WITH v AS (SELECT c, d FROM t5, w WHERE c = w.a AND c < 2) + SELECT v1.c, v1.d FROM v v1, v v2 WHERE v1.c = v2.c AND v1.d > 1 + ) x +WHERE t6.a = x.c ORDER BY 1; + a | b | c | d +---+---+---+--- + 1 | 2 | 1 | 2 +(1 row) + +CREATE TABLE t7 (f1 integer, f2 integer, f3 float); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'f1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO t7 VALUES (1, 2, 3); +INSERT INTO t7 VALUES (2, 3, 4); +INSERT INTO t7 VALUES (3, 4, 5); +INSERT INTO t7 VALUES (1, 1, 1); +INSERT INTO t7 VALUES (2, 2, 2); +INSERT INTO t7 VALUES (3, 3, 3); +INSERT INTO t7 VALUES (6, 7, 8); +INSERT INTO t7 VALUES (8, 9, NULL); +-- inlined CTEs should used the origin cexpression with recreated +explain (verbose, costs off) +with x as (select * from (select f1 from t7) ss) +select * from x where f1 = 1; + QUERY PLAN +------------------------------------------------------ + Gather Motion 1:1 (slice1; segments: 1) + Output: t7.f1 + -> Seq Scan on cte_prune.t7 + Output: t7.f1 + Filter: (t7.f1 = 1) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(7 rows) + +with x as (select * from (select f1 from t7) ss) +select * from x where f1 = 1; + f1 +---- + 1 + 1 +(2 rows) + +-- start_ignore +drop table t5; +drop table t6; +drop table t7; +drop schema cte_prune cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table t3 +drop cascades to table t4 +-- end_ignore diff --git a/src/test/regress/expected/cte_prune_optimizer.out b/src/test/regress/expected/cte_prune_optimizer.out index 831f3c9ecc8..bd8f569f691 100644 --- a/src/test/regress/expected/cte_prune_optimizer.out +++ b/src/test/regress/expected/cte_prune_optimizer.out @@ -1,6 +1,7 @@ -- start_ignore drop table if exists t1; drop table if exists t2; +SET optimizer_trace_fallback = on; -- end_ignore create table t1(v1 int, v2 int, v3 int); NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'v1' as the Apache Cloudberry data distribution key for this table. @@ -21,7 +22,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as Output: share0_ref3.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) Output: share0_ref1.v1 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=2 width=4) Output: t1.v1 Filter: (t1.v1 < 5) -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) @@ -61,7 +62,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as Output: share0_ref3.v2 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) Output: share0_ref1.v1, share0_ref1.v2 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=2 width=8) Output: t1.v1, t1.v2 Filter: (t1.v1 < 5) -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) @@ -101,7 +102,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as Output: share0_ref3.v3 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) Output: share0_ref1.v1, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=2 width=8) Output: t1.v1, t1.v3 Filter: (t1.v1 < 5) -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) @@ -142,7 +143,7 @@ explain verbose with c1 as (select * from t1) select c11.v1 from c1 as c11 left Output: share0_ref3.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=2 width=1) Output: share0_ref1.v1 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=2 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=2 width=4) Output: t1.v1 Filter: (t1.v1 < 5) -> Hash Left Join (cost=0.00..862.00 rows=2 width=4) @@ -183,7 +184,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as Output: share0_ref3.v3 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) Output: share0_ref3.v3 @@ -227,7 +228,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as Output: share0_ref3.v2 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=8) Output: t1.v1, t1.v2 -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) Output: share0_ref3.v2 @@ -262,40 +263,33 @@ with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 11 (10 rows) --- distribution col can't pruned +-- distribution col can be pruned which is better than do redistribute in CTE consumer explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=4) Output: share0_ref3.v2 -> Sequence (cost=0.00..1293.00 rows=4 width=4) Output: share0_ref3.v2 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) - Output: share0_ref1.v1, share0_ref1.v2 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) - Output: t1.v1, t1.v2 + Output: share0_ref1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: t1.v2 + Hash Key: t1.v2 + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=4) + Output: t1.v2 -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) Output: share0_ref3.v2 Hash Cond: (share0_ref3.v2 = share0_ref2.v2) - -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) Output: share0_ref3.v2 - Hash Key: share0_ref3.v2 - -> Result (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref3.v2 - -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref3.v1, share0_ref3.v2 -> Hash (cost=431.00..431.00 rows=4 width=4) Output: share0_ref2.v2 - -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=4) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) Output: share0_ref2.v2 - Hash Key: share0_ref2.v2 - -> Result (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref2.v2 - -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref2.v1, share0_ref2.v2 Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA -(29 rows) +(22 rows) with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; v2 @@ -313,38 +307,31 @@ with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 (10 rows) explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; - QUERY PLAN ------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------ Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=10 width=4) Output: share0_ref3.v3 -> Sequence (cost=0.00..1293.00 rows=4 width=4) Output: share0_ref3.v3 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) - Output: share0_ref1.v1, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) - Output: t1.v1, t1.v3 + Output: share0_ref1.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: t1.v3 + Hash Key: t1.v3 + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=4) + Output: t1.v3 -> Hash Left Join (cost=0.00..862.00 rows=4 width=4) Output: share0_ref3.v3 Hash Cond: (share0_ref3.v3 = share0_ref2.v3) - -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) Output: share0_ref3.v3 - Hash Key: share0_ref3.v3 - -> Result (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref3.v3 - -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref3.v1, share0_ref3.v3 -> Hash (cost=431.00..431.00 rows=4 width=4) Output: share0_ref2.v3 - -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=4) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=4) Output: share0_ref2.v3 - Hash Key: share0_ref2.v3 - -> Result (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref2.v3 - -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=4) - Output: share0_ref2.v1, share0_ref2.v3 Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA -(29 rows) +(22 rows) with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; v3 @@ -374,7 +361,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c Output: (sum(share0_ref3.v1)), share0_ref3.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=4) Output: t1.v1 -> GroupAggregate (cost=0.00..862.00 rows=4 width=8) Output: sum(share0_ref3.v1), share0_ref3.v1 @@ -421,7 +408,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) from c Output: (sum(share0_ref3.v1)), share0_ref3.v2 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=8) Output: t1.v1, t1.v2 -> GroupAggregate (cost=0.00..862.00 rows=4 width=8) Output: sum(share0_ref3.v1), share0_ref3.v2 @@ -471,7 +458,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v3) from c Output: (sum(share0_ref3.v3)), share0_ref3.v2 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 -> GroupAggregate (cost=0.00..862.00 rows=4 width=8) Output: sum(share0_ref3.v3), share0_ref3.v2 @@ -526,7 +513,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as Output: share0_ref3.v1, share0_ref2.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=4) Output: t1.v1 -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) Output: share0_ref3.v1, share0_ref2.v1 @@ -571,7 +558,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v1 from c1 as Output: share0_ref3.v1, share0_ref2.v3 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=8) Output: t1.v1, t1.v3 -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) Output: share0_ref3.v1, share0_ref2.v3 @@ -611,7 +598,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v1) OVER ( Output: (sum(share0_ref3.v1) OVER (?)) -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=8) Output: t1.v1, t1.v2 -> Redistribute Motion 1:3 (slice2) (cost=0.00..862.00 rows=4 width=8) Output: (sum(share0_ref3.v1) OVER (?)) @@ -661,7 +648,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER ( Output: (sum(share0_ref3.v2) OVER (?)) -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 -> Redistribute Motion 1:3 (slice2) (cost=0.00..862.00 rows=4 width=8) Output: (sum(share0_ref3.v2) OVER (?)) @@ -712,7 +699,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c Output: (sum(share1_ref2.v2)) -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=8) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=8) Output: t1.v1, t1.v2 -> Sequence (cost=0.00..2155.00 rows=7 width=8) Output: (sum(share1_ref2.v2)) @@ -795,7 +782,7 @@ explain verbose with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) from c Output: (sum(share1_ref2.v2)) -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 -> Sequence (cost=0.00..2155.00 rows=7 width=8) Output: (sum(share1_ref2.v2)) @@ -869,41 +856,35 @@ with c1 as (select v1, v2, v3 from t1) select sum(c11.v2) OVER (ORDER BY c11.v3) -- CTE producer should have right output explain verbose with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; - QUERY PLAN ----------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1724.00 rows=2 width=4) Output: share0_ref3.v1 -> Sequence (cost=0.00..1724.00 rows=1 width=4) Output: share0_ref3.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..862.00 rows=1 width=1) - Output: share0_ref1.v1, share0_ref1.v1_1 - -> Hash Join (cost=0.00..862.00 rows=1 width=8) - Output: t1.v1, t2.v1 + Output: share0_ref1.v1 + -> Hash Join (cost=0.00..862.00 rows=1 width=4) + Output: t1.v1 Hash Cond: (t1.v1 = t2.v1) - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=4) Output: t1.v1 -> Hash (cost=431.00..431.00 rows=1 width=4) Output: t2.v1 - -> Seq Scan on public.t2 (cost=0.00..431.00 rows=1 width=4) + -> Seq Scan on cte_prune.t2 (cost=0.00..431.00 rows=1 width=4) Output: t2.v1 -> Hash Left Join (cost=0.00..862.00 rows=1 width=4) Output: share0_ref3.v1 Hash Cond: (share0_ref3.v1 = share0_ref2.v1) - -> Result (cost=0.00..431.00 rows=1 width=4) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=4) Output: share0_ref3.v1 - Filter: (share0_ref3.v1 = share0_ref3.v1_1) - -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=8) - Output: share0_ref3.v1, share0_ref3.v1_1 -> Hash (cost=431.00..431.00 rows=1 width=4) Output: share0_ref2.v1 - -> Result (cost=0.00..431.00 rows=1 width=4) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=4) Output: share0_ref2.v1 - Filter: (share0_ref2.v1 = share0_ref2.v1_1) - -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=8) - Output: share0_ref2.v1, share0_ref2.v1_1 Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA -(32 rows) +(26 rows) with c1 as (select t1.v1 as v1, t2.v1 as t21, t2.v2 as t22, t2.v3 as t23 from t1 join t2 on t1.v1 = t2.v1) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; @@ -923,44 +904,37 @@ select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; explain verbose with c1 as (select sum(v1) as v1, sum(v2) as v2, v3 from t1 group by v3) select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; - QUERY PLAN -------------------------------------------------------------------------------------------------------------- + QUERY PLAN +------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.00 rows=11 width=8) Output: share0_ref3.v1 -> Sequence (cost=0.00..1293.00 rows=4 width=8) Output: share0_ref3.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=1) - Output: share0_ref1.v1, share0_ref1.v2, share0_ref1.v3 - -> HashAggregate (cost=0.00..431.00 rows=4 width=20) - Output: sum(t1.v1), sum(t1.v2), t1.v3 - Group Key: t1.v3 - -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=12) - Output: t1.v1, t1.v2, t1.v3 - Hash Key: t1.v3 - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + Output: share0_ref1.v1 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=8) + Output: (sum(t1.v1)) + Hash Key: (sum(t1.v1)) + -> HashAggregate (cost=0.00..431.00 rows=4 width=8) + Output: sum(t1.v1) + Group Key: t1.v3 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 + Hash Key: t1.v3 + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) + Output: t1.v1, t1.v2, t1.v3 -> Hash Left Join (cost=0.00..862.00 rows=4 width=8) Output: share0_ref3.v1 Hash Cond: (share0_ref3.v1 = share0_ref2.v1) - -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=8) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=8) Output: share0_ref3.v1 - Hash Key: share0_ref3.v1 - -> Result (cost=0.00..431.00 rows=4 width=8) - Output: share0_ref3.v1 - -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=4 width=8) - Output: share0_ref3.v1, share0_ref3.v2, share0_ref3.v3 -> Hash (cost=431.00..431.00 rows=4 width=8) Output: share0_ref2.v1 - -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=4 width=8) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=4 width=8) Output: share0_ref2.v1 - Hash Key: share0_ref2.v1 - -> Result (cost=0.00..431.00 rows=4 width=8) - Output: share0_ref2.v1 - -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=4 width=8) - Output: share0_ref2.v1, share0_ref2.v2, share0_ref2.v3 Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA -(35 rows) +(28 rows) with c1 as (select lt1.v3 as v3, lt1.v1 as lo1, rt1.v1 as ro1 from t1 lt1, t1 rt1 where lt1.v2 = rt1.v2 and lt1.v1 = rt1.v1) select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 from c1 where v3 > 0); @@ -968,15 +942,7 @@ select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 fro ----+----+---- (0 rows) --- cte in cte --- function call --- TPCDS cte not support reduce producter output yet --- start_ignore -drop table if exists tpcds_store_sales; -drop table if exists tpcds_date_dim; -drop table if exists tpcds_item; -drop table if exists tpcds_web_sales; --- end_ignore +-- TPCDS case create table tpcds_store_sales ( ss_sold_date_sk integer , @@ -1113,24 +1079,24 @@ explain verbose with frequent_ss_items as having count(*) >4) select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); - QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2161.00 rows=1 width=4) Output: t1.v1 -> Sequence (cost=0.00..2161.00 rows=1 width=4) Output: t1.v1 -> Shared Scan (share slice:id 1:0) (cost=0.00..868.00 rows=1 width=1) - Output: share0_ref1.itemdesc, share0_ref1.i_item_sk, share0_ref1.d_date, share0_ref1.cnt - -> Result (cost=0.00..868.00 rows=1 width=24) - Output: (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_item.i_item_sk, tpcds_date_dim.d_date, (count()) - Filter: ((count()) > 4) - -> HashAggregate (cost=0.00..868.00 rows=1 width=32) - Output: count(), count(), tpcds_date_dim.d_date, tpcds_item.i_item_sk, (substr((tpcds_item.i_item_desc)::text, 1, 30)) + Output: share0_ref1.i_item_sk + -> Result (cost=0.00..868.00 rows=1 width=4) + Output: tpcds_item.i_item_sk + Filter: ((count(*)) > 4) + -> HashAggregate (cost=0.00..868.00 rows=1 width=12) + Output: count(*), count(*), tpcds_item.i_item_sk, (substr((tpcds_item.i_item_desc)::text, 1, 30)), tpcds_date_dim.d_date Group Key: substr((tpcds_item.i_item_desc)::text, 1, 30), tpcds_item.i_item_sk, tpcds_date_dim.d_date -> Hash Join (cost=0.00..868.00 rows=1 width=16) Output: substr((tpcds_item.i_item_desc)::text, 1, 30), tpcds_date_dim.d_date, tpcds_item.i_item_sk Hash Cond: (tpcds_item.i_item_sk = tpcds_store_sales.ss_item_sk) - -> Seq Scan on public.tpcds_item (cost=0.00..431.00 rows=1 width=12) + -> Seq Scan on cte_prune.tpcds_item (cost=0.00..431.00 rows=1 width=12) Output: tpcds_item.i_item_sk, tpcds_item.i_item_desc -> Hash (cost=437.00..437.00 rows=1 width=8) Output: tpcds_store_sales.ss_item_sk, tpcds_date_dim.d_date @@ -1143,9 +1109,9 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=8) Output: tpcds_store_sales.ss_sold_date_sk, tpcds_store_sales.ss_item_sk Hash Key: tpcds_store_sales.ss_sold_date_sk - -> Seq Scan on public.tpcds_store_sales (cost=0.00..431.00 rows=1 width=8) + -> Seq Scan on cte_prune.tpcds_store_sales (cost=0.00..431.00 rows=1 width=8) Output: tpcds_store_sales.ss_sold_date_sk, tpcds_store_sales.ss_item_sk - -> Index Scan using tpcds_date_dim_pkey on public.tpcds_date_dim (cost=0.00..6.00 rows=1 width=4) + -> Index Scan using tpcds_date_dim_pkey on cte_prune.tpcds_date_dim (cost=0.00..6.00 rows=1 width=4) Output: tpcds_date_dim.d_date, tpcds_date_dim.d_year Index Cond: (tpcds_date_dim.d_date_sk = tpcds_store_sales.ss_sold_date_sk) Filter: (tpcds_date_dim.d_year = ANY ('{1999,2000,2001,2002}'::integer[])) @@ -1155,7 +1121,7 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) -> Hash Semi Join (cost=0.00..862.00 rows=1 width=4) Output: t1.v1 Hash Cond: (t1.v1 = share0_ref3.i_item_sk) - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=4) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=4) Output: t1.v1 -> Hash (cost=431.00..431.00 rows=1 width=4) Output: share0_ref3.i_item_sk @@ -1163,11 +1129,11 @@ select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) Output: share0_ref3.i_item_sk Filter: (share0_ref3.i_item_sk > 0) -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=4) - Output: share0_ref3.itemdesc, share0_ref3.i_item_sk, share0_ref3.d_date, share0_ref3.cnt + Output: share0_ref3.i_item_sk -> Hash (cost=431.00..431.00 rows=1 width=4) - Output: share0_ref2.itemdesc, share0_ref2.i_item_sk, share0_ref2.d_date, share0_ref2.cnt + Output: share0_ref2.i_item_sk -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=4) - Output: share0_ref2.itemdesc, share0_ref2.i_item_sk, share0_ref2.d_date, share0_ref2.cnt + Output: share0_ref2.i_item_sk Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA (55 rows) @@ -1179,22 +1145,22 @@ explain verbose with ws_wh as where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1730.00 rows=1 width=12) Output: t1.v1, t1.v2, t1.v3 -> Sequence (cost=0.00..1730.00 rows=1 width=12) Output: t1.v1, t1.v2, t1.v3 -> Shared Scan (share slice:id 1:0) (cost=0.00..437.00 rows=1 width=1) - Output: share0_ref1.ws_order_number, share0_ref1.ws_warehouse_sk, share0_ref1.ws_warehouse_sk_1 - -> Nested Loop (cost=0.00..437.00 rows=1 width=12) - Output: ws1.ws_order_number, ws1.ws_warehouse_sk, ws2.ws_warehouse_sk + Output: share0_ref1.ws_order_number + -> Nested Loop (cost=0.00..437.00 rows=1 width=4) + Output: ws1.ws_order_number Join Filter: true -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) Output: ws1.ws_warehouse_sk, ws1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..431.00 rows=1 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws1 (cost=0.00..431.00 rows=1 width=8) Output: ws1.ws_warehouse_sk, ws1.ws_order_number - -> Index Scan using tpcds_web_sales_pkey on public.tpcds_web_sales ws2 (cost=0.00..6.00 rows=1 width=4) + -> Index Scan using tpcds_web_sales_pkey on cte_prune.tpcds_web_sales ws2 (cost=0.00..6.00 rows=1 width=1) Output: ws2.ws_warehouse_sk Index Cond: (ws2.ws_order_number = ws1.ws_order_number) Filter: (ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) @@ -1204,7 +1170,7 @@ select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in -> Hash Semi Join (cost=0.00..862.00 rows=1 width=12) Output: t1.v1, t1.v2, t1.v3 Hash Cond: (t1.v1 = share0_ref3.ws_order_number) - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=431.00..431.00 rows=1 width=4) Output: share0_ref3.ws_order_number @@ -1215,19 +1181,17 @@ select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in Output: share0_ref3.ws_order_number Filter: (share0_ref3.ws_order_number > 0) -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=1 width=4) - Output: share0_ref3.ws_order_number, share0_ref3.ws_warehouse_sk, share0_ref3.ws_warehouse_sk_1 + Output: share0_ref3.ws_order_number -> Hash (cost=431.00..431.00 rows=1 width=4) Output: share0_ref2.ws_order_number -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=1 width=4) Output: share0_ref2.ws_order_number Hash Key: share0_ref2.ws_order_number - -> Result (cost=0.00..431.00 rows=1 width=4) + -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=1 width=4) Output: share0_ref2.ws_order_number - -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=1 width=4) - Output: share0_ref2.ws_order_number, share0_ref2.ws_warehouse_sk, share0_ref2.ws_warehouse_sk_1 Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA -(46 rows) +(44 rows) explain verbose with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 @@ -1235,22 +1199,22 @@ explain verbose with ws_wh as where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1730.00 rows=1 width=12) Output: t1.v1, t1.v2, t1.v3 -> Sequence (cost=0.00..1730.00 rows=1 width=12) Output: t1.v1, t1.v2, t1.v3 -> Shared Scan (share slice:id 1:0) (cost=0.00..437.00 rows=1 width=1) - Output: share0_ref1.ws_order_number, share0_ref1.ws_warehouse_sk, share0_ref1.ws_warehouse_sk_1 - -> Nested Loop (cost=0.00..437.00 rows=1 width=12) - Output: ws1.ws_order_number, ws1.ws_warehouse_sk, ws2.ws_warehouse_sk + Output: share0_ref1.ws_order_number, share0_ref1.ws_warehouse_sk + -> Nested Loop (cost=0.00..437.00 rows=1 width=8) + Output: ws1.ws_order_number, ws1.ws_warehouse_sk Join Filter: true -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) Output: ws1.ws_warehouse_sk, ws1.ws_order_number - -> Seq Scan on public.tpcds_web_sales ws1 (cost=0.00..431.00 rows=1 width=8) + -> Seq Scan on cte_prune.tpcds_web_sales ws1 (cost=0.00..431.00 rows=1 width=8) Output: ws1.ws_warehouse_sk, ws1.ws_order_number - -> Index Scan using tpcds_web_sales_pkey on public.tpcds_web_sales ws2 (cost=0.00..6.00 rows=1 width=4) + -> Index Scan using tpcds_web_sales_pkey on cte_prune.tpcds_web_sales ws2 (cost=0.00..6.00 rows=1 width=1) Output: ws2.ws_warehouse_sk Index Cond: (ws2.ws_order_number = ws1.ws_order_number) Filter: (ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) @@ -1260,7 +1224,7 @@ select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 -> Hash Semi Join (cost=0.00..862.00 rows=1 width=12) Output: t1.v1, t1.v2, t1.v3 Hash Cond: (t1.v1 = share0_ref3.ws_warehouse_sk) - -> Seq Scan on public.t1 (cost=0.00..431.00 rows=4 width=12) + -> Seq Scan on cte_prune.t1 (cost=0.00..431.00 rows=4 width=12) Output: t1.v1, t1.v2, t1.v3 -> Hash (cost=431.00..431.00 rows=1 width=4) Output: share0_ref3.ws_warehouse_sk @@ -1271,7 +1235,7 @@ select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 Output: share0_ref3.ws_warehouse_sk Filter: (share0_ref3.ws_order_number > 0) -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=1 width=8) - Output: share0_ref3.ws_order_number, share0_ref3.ws_warehouse_sk, share0_ref3.ws_warehouse_sk_1 + Output: share0_ref3.ws_order_number, share0_ref3.ws_warehouse_sk -> Hash (cost=431.00..431.00 rows=1 width=4) Output: share0_ref2.ws_warehouse_sk -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=1 width=4) @@ -1280,7 +1244,7 @@ select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 -> Result (cost=0.00..431.00 rows=1 width=4) Output: share0_ref2.ws_warehouse_sk -> Shared Scan (share slice:id 4:0) (cost=0.00..431.00 rows=1 width=4) - Output: share0_ref2.ws_order_number, share0_ref2.ws_warehouse_sk, share0_ref2.ws_warehouse_sk_1 + Output: share0_ref2.ws_order_number, share0_ref2.ws_warehouse_sk Settings: enable_parallel = 'off', optimizer = 'on' Optimizer: GPORCA (46 rows) @@ -1293,3 +1257,592 @@ drop table tpcds_web_sales; drop table t1; drop table t2; -- end_ignore +-- comm cases +CREATE TABLE t3 AS SELECT i as a, i+1 as b from generate_series(1,10)i; +CREATE TABLE t4 AS SELECT i as c, i+1 as d from generate_series(1,10)i; +-- Additional filtering conditions are added to the consumer. +-- This is caused by `PexprInferPredicates` in the ORCA preprocessor. +explain verbose WITH t(a,b,d) AS +( + SELECT t3.a,t3.b,t4.d FROM t3,t4 WHERE t3.a = t4.d +) +SELECT cup.*, SUM(t.d) OVER(PARTITION BY t.b) FROM + ( + SELECT t4.*, AVG(t.b) OVER(PARTITION BY t.a ORDER BY t.b desc) AS e FROM t,t4 + ) AS cup, +t WHERE cup.e < 10 +GROUP BY cup.c,cup.d, cup.e ,t.d, t.b +ORDER BY 1,2,3,4 +LIMIT 10; + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1356692546.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + Merge Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Sort (cost=0.00..1356692546.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + Sort Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Sequence (cost=0.00..1356692546.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..862.00 rows=1 width=1) + Output: share0_ref1.a, share0_ref1.b, share0_ref1.d + -> Hash Join (cost=0.00..862.00 rows=1 width=12) + Output: t3.a, t3.b, t4.d + Hash Cond: (t3.a = t4.d) + -> Seq Scan on cte_prune.t3 (cost=0.00..431.00 rows=1 width=8) + Output: t3.a, t3.b + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: t4.d + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: t4.d + -> Seq Scan on cte_prune.t4 (cost=0.00..431.00 rows=1 width=4) + Output: t4.d + -> Redistribute Motion 1:3 (slice3) (cost=0.00..1356691684.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Limit (cost=0.00..1356691684.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Gather Motion 3:1 (slice4; segments: 3) (cost=0.00..1356691684.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + Merge Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Result (cost=0.00..1356691684.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> Sort (cost=0.00..1356691684.41 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)), share0_ref2.b + Sort Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), (sum(share0_ref2.d) OVER (?)) + -> WindowAgg (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), sum(share0_ref2.d) OVER (?), share0_ref2.b + Partition By: share0_ref2.b + -> Sort (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.b, share0_ref2.d + Sort Key: share0_ref2.b + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.b, share0_ref2.d + Hash Key: share0_ref2.b + -> GroupAggregate (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.b, share0_ref2.d + Group Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.d, share0_ref2.b + -> Sort (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.b, share0_ref2.d + Sort Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.d, share0_ref2.b + -> Redistribute Motion 3:3 (slice6; segments: 3) (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.b, share0_ref2.d + Hash Key: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.d, share0_ref2.b + -> Nested Loop (cost=0.00..1356691684.40 rows=1 width=24) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)), share0_ref2.b, share0_ref2.d + Join Filter: true + -> Result (cost=0.00..1324032.22 rows=1 width=16) + Output: t4_1.c, t4_1.d, (avg(share0_ref3.b) OVER (?)) + Filter: ((avg(share0_ref3.b) OVER (?)) < '10'::numeric) + -> WindowAgg (cost=0.00..1324032.22 rows=1 width=16) + Output: avg(share0_ref3.b) OVER (?), share0_ref3.a, share0_ref3.b, t4_1.c, t4_1.d + Partition By: share0_ref3.a + Order By: share0_ref3.b + -> Sort (cost=0.00..1324032.22 rows=1 width=16) + Output: share0_ref3.a, share0_ref3.b, t4_1.c, t4_1.d + Sort Key: share0_ref3.a, share0_ref3.b DESC + -> Redistribute Motion 3:3 (slice8; segments: 3) (cost=0.00..1324032.22 rows=1 width=16) + Output: share0_ref3.a, share0_ref3.b, t4_1.c, t4_1.d + Hash Key: share0_ref3.a + -> Nested Loop (cost=0.00..1324032.22 rows=1 width=16) + Output: share0_ref3.a, share0_ref3.b, t4_1.c, t4_1.d + Join Filter: true + -> Broadcast Motion 3:3 (slice9; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref3.a, share0_ref3.b + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref3.a, share0_ref3.b + Filter: (share0_ref3.a = share0_ref3.d) + -> Shared Scan (share slice:id 9:0) (cost=0.00..431.00 rows=1 width=12) + Output: share0_ref3.a, share0_ref3.b, share0_ref3.d + -> Seq Scan on cte_prune.t4 t4_1 (cost=0.00..431.00 rows=1 width=8) + Output: t4_1.c, t4_1.d + -> Materialize (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.b, share0_ref2.d + -> Broadcast Motion 3:3 (slice7; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.b, share0_ref2.d + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.b, share0_ref2.d + Filter: (share0_ref2.a = share0_ref2.d) + -> Shared Scan (share slice:id 7:0) (cost=0.00..431.00 rows=1 width=12) + Output: share0_ref2.a, share0_ref2.b, share0_ref2.d + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(90 rows) + +WITH t(a,b,d) AS +( + SELECT t3.a,t3.b,t4.d FROM t3,t4 WHERE t3.a = t4.d +) +SELECT cup.*, SUM(t.d) OVER(PARTITION BY t.b) FROM + ( + SELECT t4.*, AVG(t.b) OVER(PARTITION BY t.a ORDER BY t.b desc) AS e FROM t,t4 + ) AS cup, +t WHERE cup.e < 10 +GROUP BY cup.c,cup.d, cup.e ,t.d, t.b +ORDER BY 1,2,3,4 +LIMIT 10; + c | d | e | sum +---+---+--------------------+----- + 1 | 2 | 3.0000000000000000 | 140 + 1 | 2 | 3.0000000000000000 | 210 + 1 | 2 | 3.0000000000000000 | 280 + 1 | 2 | 3.0000000000000000 | 350 + 1 | 2 | 3.0000000000000000 | 420 + 1 | 2 | 3.0000000000000000 | 490 + 1 | 2 | 3.0000000000000000 | 560 + 1 | 2 | 3.0000000000000000 | 630 + 1 | 2 | 3.0000000000000000 | 700 + 1 | 2 | 4.0000000000000000 | 140 +(10 rows) + +-- grouping set will generate the internal CTE. +CREATE TABLE cte_prune_tenk1 (unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'unique1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO cte_prune_tenk1 +SELECT + i AS unique1, + (i + 10000) AS unique2, + i % 2 AS two, + i % 4 AS four, + i % 10 AS ten, + i % 20 AS twenty, + i % 100 AS hundred, + i % 1000 AS thousand, + i % 2000 AS twothousand, + i % 5000 AS fivethous, + i % 10000 AS tenthous, + (2 * i + 1) AS odd, + (2 * i) AS even, + ('A' || lpad(i::text, 4, '0'))::name AS stringu1, + ('B' || lpad(i::text, 4, '0'))::name AS stringu2, + (CASE (i % 4) + WHEN 0 THEN 'AAAA'::name + WHEN 1 THEN 'BBBB'::name + WHEN 2 THEN 'CCCC'::name + ELSE 'DDDD'::name + END) AS string4 +FROM generate_series(0, 99) AS i; +INFO: GPORCA failed to produce a plan, falling back to Postgres-based planner +DETAIL: Falling back to Postgres-based planner because GPORCA does not support the following feature: Non-default collation +explain verbose select four, x + from (select four, ten, 'foo'::text as x from cte_prune_tenk1) as t + group by grouping sets (four, x) + having x = 'foo'; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.00 rows=1 width=12) + Output: (NULL::integer), share0_ref2.x + -> Sequence (cost=0.00..862.00 rows=1 width=12) + Output: (NULL::integer), share0_ref2.x + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=1 width=1) + Output: share0_ref1.four, share0_ref1.x + -> Seq Scan on cte_prune.cte_prune_tenk1 (cost=0.00..431.00 rows=1 width=4) + Output: cte_prune_tenk1.four, 'foo'::text + -> Append (cost=0.00..431.00 rows=1 width=12) + -> GroupAggregate (cost=0.00..431.00 rows=1 width=8) + Output: NULL::integer, share0_ref2.x + Group Key: share0_ref2.x + -> Sort (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.x + Sort Key: share0_ref2.x + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.x + Hash Key: share0_ref2.x + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.x + Filter: (share0_ref2.x = 'foo'::text) + -> Shared Scan (share slice:id 2:0) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref2.four, share0_ref2.x + -> Result (cost=0.00..0.00 rows=0 width=12) + Output: (NULL::integer), (NULL::text) + One-Time Filter: (gp_execution_segment() = 0) + -> Result (cost=0.00..0.00 rows=0 width=12) + Output: NULL::integer, NULL::text + One-Time Filter: false + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(31 rows) + +select four, x + from (select four, ten, 'foo'::text as x from cte_prune_tenk1) as t + group by grouping sets (four, x) + having x = 'foo'; + four | x +------+----- + | foo +(1 row) + +-- nest CTE cases +-- start_ignore +drop table city; +ERROR: table "city" does not exist +drop table country; +ERROR: table "country" does not exist +drop table countrylanguage; +ERROR: table "countrylanguage" does not exist +-- end_ignore +CREATE TABLE city ( + id integer NOT NULL, + name text NOT NULL, + countrycode character(3) NOT NULL, + district text NOT NULL, + population integer NOT NULL +) distributed by(id); +CREATE TABLE country ( + code character(3) NOT NULL, + name text NOT NULL, + continent text NOT NULL, + region text NOT NULL, + surfacearea numeric(10,2) NOT NULL, + indepyear smallint, + population integer NOT NULL, + lifeexpectancy real, + gnp numeric(10,2), + gnpold numeric(10,2), + localname text NOT NULL, + governmentform text NOT NULL, + headofstate text, + capital integer, + code2 character(2) NOT NULL +) distributed by (code); +CREATE TABLE countrylanguage ( + countrycode character(3) NOT NULL, + "language" text NOT NULL, + isofficial boolean NOT NULL, + percentage real NOT NULL +)distributed by (countrycode,language); +ALTER TABLE ONLY city + ADD CONSTRAINT city_pkey PRIMARY KEY (id); +ALTER TABLE ONLY country + ADD CONSTRAINT country_pkey PRIMARY KEY (code); +ALTER TABLE ONLY countrylanguage + ADD CONSTRAINT countrylanguage_pkey PRIMARY KEY (countrycode, "language"); +-- CTE1(inlined) in CTE2(no-inlined) case +explain verbose with country as +(select country.code,country.name COUNTRY, city.name CAPITAL, language, isofficial, percentage + FROM country,city,countrylanguage + WHERE country.code = countrylanguage.countrycode + and country.capital = city.id + and country.continent = 'Europe'), +countrylanguage as +(select country.code,country.COUNTRY,country.language,country.isofficial,country.percentage + FROM country,countrylanguage + WHERE country.code = countrylanguage.countrycode +) +select * from +(select * from country where isofficial='True') country, +(select * from countrylanguage where percentage > 50) countrylanguage +where country.percentage = countrylanguage.percentage order by countrylanguage.COUNTRY,country.language LIMIT 40; + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1736.01 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Merge Key: share0_ref2.name, share0_ref3.language + -> Sort (cost=0.00..1736.01 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Sort Key: share0_ref2.name, share0_ref3.language + -> Sequence (cost=0.00..1736.00 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Shared Scan (share slice:id 1:0) (cost=0.00..868.00 rows=1 width=1) + Output: share0_ref1.code, share0_ref1.name, share0_ref1.name_1, share0_ref1.language, share0_ref1.isofficial, share0_ref1.percentage + -> Hash Join (cost=0.00..868.00 rows=1 width=37) + Output: country.code, country.name, city.name, countrylanguage.language, countrylanguage.isofficial, countrylanguage.percentage + Hash Cond: (country.capital = city.id) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..437.00 rows=1 width=33) + Output: country.code, country.name, country.capital, countrylanguage.language, countrylanguage.isofficial, countrylanguage.percentage + Hash Key: country.capital + -> Nested Loop (cost=0.00..437.00 rows=1 width=33) + Output: country.code, country.name, country.capital, countrylanguage.language, countrylanguage.isofficial, countrylanguage.percentage + Join Filter: true + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=21) + Output: countrylanguage.countrycode, countrylanguage.language, countrylanguage.isofficial, countrylanguage.percentage + Hash Key: countrylanguage.countrycode + -> Seq Scan on cte_prune.countrylanguage (cost=0.00..431.00 rows=1 width=21) + Output: countrylanguage.countrycode, countrylanguage.language, countrylanguage.isofficial, countrylanguage.percentage + -> Index Scan using country_pkey on cte_prune.country (cost=0.00..6.00 rows=1 width=20) + Output: country.code, country.name, country.continent, country.capital + Index Cond: (country.code = countrylanguage.countrycode) + Filter: (country.continent = 'Europe'::text) + -> Hash (cost=431.00..431.00 rows=1 width=12) + Output: city.id, city.name + -> Seq Scan on cte_prune.city (cost=0.00..431.00 rows=1 width=12) + Output: city.id, city.name + -> Redistribute Motion 1:3 (slice4) (cost=0.00..868.00 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Limit (cost=0.00..868.00 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Gather Motion 3:1 (slice5; segments: 3) (cost=0.00..868.00 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Merge Key: share0_ref2.name, share0_ref3.language + -> Sort (cost=0.00..868.00 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Sort Key: share0_ref2.name, share0_ref3.language + -> Hash Join (cost=0.00..868.00 rows=1 width=66) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage, share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Hash Cond: (share0_ref3.percentage = share0_ref2.percentage) + -> Redistribute Motion 3:3 (slice6; segments: 3) (cost=0.00..431.00 rows=1 width=37) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage + Hash Key: share0_ref3.percentage + -> Result (cost=0.00..431.00 rows=1 width=37) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage + Filter: (share0_ref3.isofficial AND (share0_ref3.percentage > '50'::double precision)) + -> Shared Scan (share slice:id 6:0) (cost=0.00..431.00 rows=1 width=37) + Output: share0_ref3.code, share0_ref3.name, share0_ref3.name_1, share0_ref3.language, share0_ref3.isofficial, share0_ref3.percentage + -> Hash (cost=437.00..437.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Redistribute Motion 3:3 (slice7; segments: 3) (cost=0.00..437.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Hash Key: share0_ref2.percentage + -> Result (cost=0.00..437.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Filter: (share0_ref2.percentage > '50'::double precision) + -> Nested Loop (cost=0.00..437.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + Join Filter: true + -> Broadcast Motion 3:3 (slice8; segments: 3) (cost=0.00..431.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Result (cost=0.00..431.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Shared Scan (share slice:id 8:0) (cost=0.00..431.00 rows=1 width=29) + Output: share0_ref2.code, share0_ref2.name, share0_ref2.name_1, share0_ref2.language, share0_ref2.isofficial, share0_ref2.percentage + -> Index Scan using countrylanguage_pkey on cte_prune.countrylanguage countrylanguage_1 (cost=0.00..6.00 rows=1 width=1) + Index Cond: (countrylanguage_1.countrycode = share0_ref2.code) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(74 rows) + +-- CTE in the main query and subqueries within the main query +explain verbose with bad_headofstates as +( + select country.code,country.name,country.headofstate,countrylanguage.language + from + country,countrylanguage + where country.code = countrylanguage.countrycode and countrylanguage.isofficial=true + and (country.gnp < country.gnpold or country.gnp < 3000) +) +select OUTERMOST_FOO.*,bad_headofstates.headofstate from ( +select avg(population),region from +( +select FOO.*,bad_headofstates.headofstate,city.name +from +(select bad_headofstates.code,country.capital,country.region,country.population from +bad_headofstates,country where bad_headofstates.code = country.code) FOO, bad_headofstates,city +where FOO.code = bad_headofstates.code and FOO.capital = city.id) OUTER_FOO +group by region ) OUTERMOST_FOO,bad_headofstates,country +where country.code = bad_headofstates.code and country.region = OUTERMOST_FOO.region +order by OUTERMOST_FOO.region,bad_headofstates.headofstate LIMIT 40; + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------- + Sequence (cost=0.00..1748.00 rows=1 width=24) + Output: (avg(country_1.population)), country_1.region, share0_ref2.headofstate + -> Shared Scan (share slice:id 0:0) (cost=0.00..437.00 rows=1 width=1) + Output: share0_ref1.code, share0_ref1.headofstate + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..437.00 rows=1 width=16) + Output: country.code, country.headofstate + -> Nested Loop (cost=0.00..437.00 rows=1 width=16) + Output: country.code, country.headofstate + Join Filter: true + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: countrylanguage.countrycode + Hash Key: countrylanguage.countrycode + -> Seq Scan on cte_prune.countrylanguage (cost=0.00..431.00 rows=1 width=8) + Output: countrylanguage.countrycode + Filter: countrylanguage.isofficial + -> Index Scan using country_pkey on cte_prune.country (cost=0.00..6.00 rows=1 width=16) + Output: country.code, country.gnp, country.gnpold, country.headofstate + Index Cond: (country.code = countrylanguage.countrycode) + Filter: ((country.gnp < country.gnpold) OR (country.gnp < '3000'::numeric)) + -> Limit (cost=0.00..1311.00 rows=1 width=24) + Output: (avg(country_1.population)), country_1.region, share0_ref2.headofstate + -> Sort (cost=0.00..1311.00 rows=1 width=24) + Output: (avg(country_1.population)), country_1.region, share0_ref2.headofstate + Sort Key: country_1.region, share0_ref2.headofstate + -> Hash Join (cost=0.00..1311.00 rows=1 width=24) + Output: (avg(country_1.population)), country_1.region, share0_ref2.headofstate + Hash Cond: (country_1.region = country_2.region) + -> HashAggregate (cost=0.00..874.00 rows=1 width=16) + Output: avg(country_1.population), country_1.region + Group Key: country_1.region + -> Hash Join (cost=0.00..874.00 rows=1 width=12) + Output: country_1.region, country_1.population + Hash Cond: (share0_ref4.code = share0_ref3.code) + -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..443.00 rows=1 width=20) + Output: share0_ref4.code, country_1.region, country_1.population + -> Nested Loop (cost=0.00..443.00 rows=1 width=20) + Output: share0_ref4.code, country_1.region, country_1.population + Join Filter: true + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..437.00 rows=1 width=24) + Output: share0_ref4.code, country_1.region, country_1.population, country_1.capital + Hash Key: country_1.capital + -> Nested Loop (cost=0.00..437.00 rows=1 width=24) + Output: share0_ref4.code, country_1.region, country_1.population, country_1.capital + Join Filter: true + -> Redistribute Motion 1:3 (slice5) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref4.code + Hash Key: share0_ref4.code + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref4.code + -> Shared Scan (share slice:id 5:0) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref4.code, share0_ref4.headofstate + -> Index Scan using country_pkey on cte_prune.country country_1 (cost=0.00..6.00 rows=1 width=16) + Output: country_1.region, country_1.population, country_1.capital + Index Cond: (country_1.code = share0_ref4.code) + -> Index Scan using city_pkey on cte_prune.city (cost=0.00..6.00 rows=1 width=1) + Index Cond: (city.id = country_1.capital) + -> Hash (cost=431.00..431.00 rows=1 width=8) + Output: share0_ref3.code, share0_ref3.headofstate + -> Shared Scan (share slice:id 0:0) (cost=0.00..431.00 rows=1 width=8) + Output: share0_ref3.code, share0_ref3.headofstate + -> Hash (cost=437.00..437.00 rows=1 width=16) + Output: share0_ref2.headofstate, country_2.region + -> Gather Motion 3:1 (slice6; segments: 3) (cost=0.00..437.00 rows=1 width=16) + Output: share0_ref2.headofstate, country_2.region + -> Nested Loop (cost=0.00..437.00 rows=1 width=16) + Output: share0_ref2.headofstate, country_2.region + Join Filter: true + -> Redistribute Motion 1:3 (slice7) (cost=0.00..431.00 rows=1 width=16) + Output: share0_ref2.code, share0_ref2.headofstate + Hash Key: share0_ref2.code + -> Shared Scan (share slice:id 7:0) (cost=0.00..431.00 rows=1 width=16) + Output: share0_ref2.code, share0_ref2.headofstate + -> Index Scan using country_pkey on cte_prune.country country_2 (cost=0.00..6.00 rows=1 width=8) + Output: country_2.region + Index Cond: (country_2.code = share0_ref2.code) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(77 rows) + +-- start_ignore +drop table city; +drop table country; +drop table countrylanguage; +-- end_ignore +-- inlined CTEs +CREATE TABLE t5 AS SELECT i as c, i+1 as d from generate_series(1,10)i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'c' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +CREATE TABLE t6 AS SELECT i as a, i+1 as b from generate_series(1,10)i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- inlined CTEs should have not unused columns(ex. t5.*, t6.* in output) +explain verbose WITH w AS (SELECT a, b from t6 where b < 5) +SELECT * +FROM t6, + (WITH v AS (SELECT c, d FROM t5, w WHERE c = w.a AND c < 2) + SELECT v1.c, v1.d FROM v v1, v v2 WHERE v1.c = v2.c AND v1.d > 1 + ) x +WHERE t6.a = x.c ORDER BY 1; + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2155.00 rows=1 width=16) + Output: t6_1.a, t6_1.b, share1_ref3.c, share1_ref3.d + Merge Key: t6_1.a + -> Sort (cost=0.00..2155.00 rows=1 width=16) + Output: t6_1.a, t6_1.b, share1_ref3.c, share1_ref3.d + Sort Key: t6_1.a + -> Hash Join (cost=0.00..2155.00 rows=1 width=16) + Output: t6_1.a, t6_1.b, share1_ref3.c, share1_ref3.d + Hash Cond: (share1_ref3.c = t6_1.a) + -> Sequence (cost=0.00..1724.00 rows=1 width=8) + Output: share1_ref3.c, share1_ref3.d + -> Shared Scan (share slice:id 1:1) (cost=0.00..862.00 rows=1 width=1) + Output: share1_ref1.c, share1_ref1.d + -> Hash Join (cost=0.00..862.00 rows=1 width=8) + Output: t5.c, t5.d + Hash Cond: (t5.c = t6.a) + -> Seq Scan on cte_prune.t5 (cost=0.00..431.00 rows=1 width=8) + Output: t5.c, t5.d + Filter: (t5.c < 2) + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: t6.a + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: t6.a + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: t6.a + Filter: (t6.a < 2) + -> Seq Scan on cte_prune.t6 (cost=0.00..431.00 rows=1 width=4) + Output: t6.a + Filter: (t6.b < 5) + -> Hash Join (cost=0.00..862.00 rows=1 width=8) + Output: share1_ref3.c, share1_ref3.d + Hash Cond: (share1_ref3.c = share1_ref2.c) + -> Result (cost=0.00..431.00 rows=1 width=8) + Output: share1_ref3.c, share1_ref3.d + Filter: ((share1_ref3.d > 1) AND (share1_ref3.c < 2)) + -> Shared Scan (share slice:id 1:1) (cost=0.00..431.00 rows=1 width=8) + Output: share1_ref3.c, share1_ref3.d + -> Hash (cost=431.00..431.00 rows=1 width=4) + Output: share1_ref2.c + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=4) + Output: share1_ref2.c + -> Result (cost=0.00..431.00 rows=1 width=4) + Output: share1_ref2.c + Filter: (share1_ref2.c < 2) + -> Shared Scan (share slice:id 3:1) (cost=0.00..431.00 rows=1 width=4) + Output: share1_ref2.c, share1_ref2.d + -> Hash (cost=431.00..431.00 rows=1 width=8) + Output: t6_1.a, t6_1.b + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=1 width=8) + Output: t6_1.a, t6_1.b + -> Seq Scan on cte_prune.t6 t6_1 (cost=0.00..431.00 rows=1 width=8) + Output: t6_1.a, t6_1.b + Filter: (t6_1.a < 2) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(55 rows) + +WITH w AS (SELECT a, b from t6 where b < 5) +SELECT * +FROM t6, + (WITH v AS (SELECT c, d FROM t5, w WHERE c = w.a AND c < 2) + SELECT v1.c, v1.d FROM v v1, v v2 WHERE v1.c = v2.c AND v1.d > 1 + ) x +WHERE t6.a = x.c ORDER BY 1; + a | b | c | d +---+---+---+--- + 1 | 2 | 1 | 2 +(1 row) + +CREATE TABLE t7 (f1 integer, f2 integer, f3 float); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'f1' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +INSERT INTO t7 VALUES (1, 2, 3); +INSERT INTO t7 VALUES (2, 3, 4); +INSERT INTO t7 VALUES (3, 4, 5); +INSERT INTO t7 VALUES (1, 1, 1); +INSERT INTO t7 VALUES (2, 2, 2); +INSERT INTO t7 VALUES (3, 3, 3); +INSERT INTO t7 VALUES (6, 7, 8); +INSERT INTO t7 VALUES (8, 9, NULL); +-- inlined CTEs should used the origin cexpression with recreated +explain (verbose, costs off) +with x as (select * from (select f1 from t7) ss) +select * from x where f1 = 1; + QUERY PLAN +----------------------------------------------------- + Gather Motion 1:1 (slice1; segments: 1) + Output: f1 + -> Seq Scan on cte_prune.t7 + Output: f1 + Filter: (t7.f1 = 1) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(7 rows) + +with x as (select * from (select f1 from t7) ss) +select * from x where f1 = 1; + f1 +---- + 1 + 1 +(2 rows) + +-- start_ignore +drop table t5; +drop table t6; +drop table t7; +drop schema cte_prune cascade; +NOTICE: drop cascades to 2 other objects +DETAIL: drop cascades to table t3 +drop cascades to table t4 +-- end_ignore diff --git a/src/test/regress/expected/gp_dqa_optimizer.out b/src/test/regress/expected/gp_dqa_optimizer.out index 394cf7ce670..09af6999902 100644 --- a/src/test/regress/expected/gp_dqa_optimizer.out +++ b/src/test/regress/expected/gp_dqa_optimizer.out @@ -3269,8 +3269,8 @@ DETAIL: Falling back to Postgres-based planner because GPORCA does not support set optimizer_enable_multiple_distinct_aggs=on; explain (verbose on, costs off) select count(distinct a), count(distinct b) from dqa_f4 group by c; - QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + QUERY PLAN +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) Output: (count(DISTINCT share0_ref3.a)), (count(DISTINCT share0_ref2.b)) -> Sequence @@ -3294,7 +3294,7 @@ explain (verbose on, costs off) select count(distinct a), count(distinct b) from -> Result Output: share0_ref3.a, share0_ref3.c -> Shared Scan (share slice:id 2:0) - Output: share0_ref3.a, share0_ref3.b, share0_ref3.c, share0_ref3.ctid, share0_ref3.xmin, share0_ref3.cmin, share0_ref3.xmax, share0_ref3.cmax, share0_ref3.tableoid, share0_ref3.gp_segment_id, share0_ref3.gp_foreign_server + Output: share0_ref3.a, share0_ref3.b, share0_ref3.c -> Hash Output: (count(DISTINCT share0_ref2.b)), share0_ref2.c -> GroupAggregate @@ -3309,9 +3309,9 @@ explain (verbose on, costs off) select count(distinct a), count(distinct b) from -> Result Output: share0_ref2.b, share0_ref2.c -> Shared Scan (share slice:id 3:0) - Output: share0_ref2.a, share0_ref2.b, share0_ref2.c, share0_ref2.ctid, share0_ref2.xmin, share0_ref2.cmin, share0_ref2.xmax, share0_ref2.cmax, share0_ref2.tableoid, share0_ref2.gp_segment_id, share0_ref2.gp_foreign_server + Output: share0_ref2.a, share0_ref2.b, share0_ref2.c Settings: enable_groupagg = 'off', enable_hashagg = 'on', enable_parallel = 'off', gp_motion_cost_per_row = '2', optimizer = 'on' - Optimizer: Pivotal Optimizer (GPORCA) + Optimizer: GPORCA (41 rows) select count(distinct a), count(distinct b) from dqa_f4 group by c; diff --git a/src/test/regress/expected/gporca_optimizer.out b/src/test/regress/expected/gporca_optimizer.out index 6afe28abe77..a1a315a1d6e 100644 --- a/src/test/regress/expected/gporca_optimizer.out +++ b/src/test/regress/expected/gporca_optimizer.out @@ -11295,10 +11295,9 @@ EXPLAIN (COSTS OFF) WITH abc AS (SELECT onetimefilter1.a, onetimefilter1.b FROM Filter: (share0_ref3.b = f1.b) -> Materialize -> Broadcast Motion 3:3 (slice6; segments: 3) - -> Result - -> Shared Scan (share slice:id 6:0) + -> Shared Scan (share slice:id 6:0) Optimizer: GPORCA -(43 rows) +(42 rows) WITH abc AS (SELECT onetimefilter1.a, onetimefilter1.b FROM onetimefilter1, onetimefilter2 WHERE onetimefilter1.a=onetimefilter2.a) SELECT (SELECT 1 FROM abc WHERE f1.b = f2.b LIMIT 1), COALESCE((SELECT 2 FROM abc WHERE f1.a=random() AND f1.a=2), 0), (SELECT b FROM abc WHERE b=f1.b) FROM onetimefilter1 f1, onetimefilter2 f2 WHERE f1.b = f2.b; ?column? | coalesce | b @@ -12013,35 +12012,31 @@ EXPLAIN SELECT a, b FROM atab_old_hash FULL JOIN btab_old_hash ON a |=| b; -> Result (cost=0.00..2586.00 rows=4 width=8) -> Sequence (cost=0.00..2586.00 rows=4 width=8) -> Shared Scan (share slice:id 1:2) (cost=0.00..431.00 rows=2 width=1) - -> Seq Scan on btab_old_hash (cost=0.00..431.00 rows=2 width=34) + -> Seq Scan on btab_old_hash (cost=0.00..431.00 rows=2 width=38) -> Sequence (cost=0.00..2155.00 rows=4 width=8) -> Shared Scan (share slice:id 1:3) (cost=0.00..431.00 rows=1 width=1) - -> Seq Scan on atab_old_hash (cost=0.00..431.00 rows=1 width=34) + -> Seq Scan on atab_old_hash (cost=0.00..431.00 rows=1 width=38) -> Append (cost=0.00..1724.00 rows=4 width=8) -> Hash Left Join (cost=0.00..862.00 rows=3 width=8) Hash Cond: (share2_ref2.b |=| share3_ref2.a) -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=2 width=4) Hash Key: share2_ref2.b - -> Result (cost=0.00..431.00 rows=2 width=4) - -> Shared Scan (share slice:id 2:2) (cost=0.00..431.00 rows=2 width=4) + -> Shared Scan (share slice:id 2:2) (cost=0.00..431.00 rows=2 width=4) -> Hash (cost=431.00..431.00 rows=1 width=4) -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=1 width=4) Hash Key: share3_ref2.a - -> Result (cost=0.00..431.00 rows=1 width=4) - -> Shared Scan (share slice:id 3:3) (cost=0.00..431.00 rows=1 width=4) + -> Shared Scan (share slice:id 3:3) (cost=0.00..431.00 rows=1 width=4) -> Hash Anti Join (cost=0.00..862.00 rows=1 width=4) Hash Cond: (share3_ref3.a |=| share2_ref3.b) -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=1 width=4) Hash Key: share3_ref3.a - -> Result (cost=0.00..431.00 rows=1 width=4) - -> Shared Scan (share slice:id 4:3) (cost=0.00..431.00 rows=1 width=4) + -> Shared Scan (share slice:id 4:3) (cost=0.00..431.00 rows=1 width=4) -> Hash (cost=431.00..431.00 rows=2 width=4) -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..431.00 rows=2 width=4) Hash Key: share2_ref3.b - -> Result (cost=0.00..431.00 rows=2 width=4) - -> Shared Scan (share slice:id 5:2) (cost=0.00..431.00 rows=2 width=4) + -> Shared Scan (share slice:id 5:2) (cost=0.00..431.00 rows=2 width=4) Optimizer: GPORCA -(32 rows) +(28 rows) SELECT a, b FROM atab_old_hash FULL JOIN btab_old_hash ON a |=| b; a | b diff --git a/src/test/regress/expected/shared_scan_optimizer.out b/src/test/regress/expected/shared_scan_optimizer.out index d782988bbd6..a256c71063a 100644 --- a/src/test/regress/expected/shared_scan_optimizer.out +++ b/src/test/regress/expected/shared_scan_optimizer.out @@ -107,11 +107,12 @@ WITH cte AS (SELECT * FROM t1 WHERE random() < 0.1 LIMIT 10) SELECT a, 1, 1 FROM Hash Cond: (share0_ref2.a = t2.a) -> Redistribute Motion 1:3 (slice3) Hash Key: share0_ref2.a - -> Shared Scan (share slice:id 3:0) + -> Result + -> Shared Scan (share slice:id 3:0) -> Hash -> Seq Scan on t2 Optimizer: GPORCA -(16 rows) +(17 rows) -- This functions returns one more column than expected. CREATE OR REPLACE FUNCTION col_mismatch_func1() RETURNS TABLE (field1 int, field2 int) diff --git a/src/test/regress/parallel_schedule b/src/test/regress/parallel_schedule index 97274fdf8ba..48d6843b8a3 100644 --- a/src/test/regress/parallel_schedule +++ b/src/test/regress/parallel_schedule @@ -158,7 +158,7 @@ test: json jsonb json_encoding jsonpath jsonpath_encoding jsonb_jsonpath # NB: temp.sql does a reconnect which transiently uses 2 connections, # so keep this parallel group to at most 19 tests # ---------- -test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion truncate alter_table sequence polymorphism rowtypes returning with xml +test: plancache limit plpgsql copy2 temp domain rangefuncs prepare conversion truncate alter_table sequence polymorphism rowtypes returning with xml cte_prune # large objects are not supported by GPDB # test: largeobject diff --git a/src/test/regress/sql/cte_prune.sql b/src/test/regress/sql/cte_prune.sql index 2c16b515a61..a87129d5e50 100644 --- a/src/test/regress/sql/cte_prune.sql +++ b/src/test/regress/sql/cte_prune.sql @@ -1,6 +1,7 @@ -- start_ignore -drop table if exists t1; -drop table if exists t2; +create schema cte_prune; +set search_path = cte_prune; +SET optimizer_trace_fallback = on; -- end_ignore create table t1(v1 int, v2 int, v3 int); @@ -29,7 +30,7 @@ with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v1=c22.v2; --- distribution col can't pruned +-- distribution col can be pruned which is better than do redistribute in CTE consumer explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; with c1 as (select v1, v2, v3 from t1) select c11.v2 from c1 as c11 left join c1 as c22 on c11.v2=c22.v2; explain verbose with c1 as (select v1, v2, v3 from t1) select c11.v3 from c1 as c11 left join c1 as c22 on c11.v3=c22.v3; @@ -81,20 +82,7 @@ select c11.v1 from c1 as c11 left join c1 as c22 on c11.v1=c22.v1; with c1 as (select lt1.v3 as v3, lt1.v1 as lo1, rt1.v1 as ro1 from t1 lt1, t1 rt1 where lt1.v2 = rt1.v2 and lt1.v1 = rt1.v1) select * from t1 where t1.v1 in (select v3 from c1) and t1.v1 in (select v3 from c1 where v3 > 0); --- cte in cte - - --- function call - - --- TPCDS cte not support reduce producter output yet - --- start_ignore -drop table if exists tpcds_store_sales; -drop table if exists tpcds_date_dim; -drop table if exists tpcds_item; -drop table if exists tpcds_web_sales; --- end_ignore +-- TPCDS case create table tpcds_store_sales ( @@ -261,3 +249,217 @@ drop table tpcds_web_sales; drop table t1; drop table t2; -- end_ignore + + +-- comm cases + +CREATE TABLE t3 AS SELECT i as a, i+1 as b from generate_series(1,10)i; +CREATE TABLE t4 AS SELECT i as c, i+1 as d from generate_series(1,10)i; + +-- Additional filtering conditions are added to the consumer. +-- This is caused by `PexprInferPredicates` in the ORCA preprocessor. +explain verbose WITH t(a,b,d) AS +( + SELECT t3.a,t3.b,t4.d FROM t3,t4 WHERE t3.a = t4.d +) +SELECT cup.*, SUM(t.d) OVER(PARTITION BY t.b) FROM + ( + SELECT t4.*, AVG(t.b) OVER(PARTITION BY t.a ORDER BY t.b desc) AS e FROM t,t4 + ) AS cup, +t WHERE cup.e < 10 +GROUP BY cup.c,cup.d, cup.e ,t.d, t.b +ORDER BY 1,2,3,4 +LIMIT 10; + +WITH t(a,b,d) AS +( + SELECT t3.a,t3.b,t4.d FROM t3,t4 WHERE t3.a = t4.d +) +SELECT cup.*, SUM(t.d) OVER(PARTITION BY t.b) FROM + ( + SELECT t4.*, AVG(t.b) OVER(PARTITION BY t.a ORDER BY t.b desc) AS e FROM t,t4 + ) AS cup, +t WHERE cup.e < 10 +GROUP BY cup.c,cup.d, cup.e ,t.d, t.b +ORDER BY 1,2,3,4 +LIMIT 10; + +-- grouping set will generate the internal CTE. + +CREATE TABLE cte_prune_tenk1 (unique1 int4, unique2 int4, two int4, four int4, ten int4, twenty int4, hundred int4, thousand int4, twothousand int4, fivethous int4, tenthous int4, odd int4, even int4, stringu1 name, stringu2 name, string4 name); +INSERT INTO cte_prune_tenk1 +SELECT + i AS unique1, + (i + 10000) AS unique2, + i % 2 AS two, + i % 4 AS four, + i % 10 AS ten, + i % 20 AS twenty, + i % 100 AS hundred, + i % 1000 AS thousand, + i % 2000 AS twothousand, + i % 5000 AS fivethous, + i % 10000 AS tenthous, + (2 * i + 1) AS odd, + (2 * i) AS even, + ('A' || lpad(i::text, 4, '0'))::name AS stringu1, + ('B' || lpad(i::text, 4, '0'))::name AS stringu2, + (CASE (i % 4) + WHEN 0 THEN 'AAAA'::name + WHEN 1 THEN 'BBBB'::name + WHEN 2 THEN 'CCCC'::name + ELSE 'DDDD'::name + END) AS string4 +FROM generate_series(0, 99) AS i; + +explain verbose select four, x + from (select four, ten, 'foo'::text as x from cte_prune_tenk1) as t + group by grouping sets (four, x) + having x = 'foo'; + +select four, x + from (select four, ten, 'foo'::text as x from cte_prune_tenk1) as t + group by grouping sets (four, x) + having x = 'foo'; + +-- nest CTE cases + +-- start_ignore +drop table city; +drop table country; +drop table countrylanguage; +-- end_ignore + +CREATE TABLE city ( + id integer NOT NULL, + name text NOT NULL, + countrycode character(3) NOT NULL, + district text NOT NULL, + population integer NOT NULL +) distributed by(id); + +CREATE TABLE country ( + code character(3) NOT NULL, + name text NOT NULL, + continent text NOT NULL, + region text NOT NULL, + surfacearea numeric(10,2) NOT NULL, + indepyear smallint, + population integer NOT NULL, + lifeexpectancy real, + gnp numeric(10,2), + gnpold numeric(10,2), + localname text NOT NULL, + governmentform text NOT NULL, + headofstate text, + capital integer, + code2 character(2) NOT NULL +) distributed by (code); + +CREATE TABLE countrylanguage ( + countrycode character(3) NOT NULL, + "language" text NOT NULL, + isofficial boolean NOT NULL, + percentage real NOT NULL +)distributed by (countrycode,language); + +ALTER TABLE ONLY city + ADD CONSTRAINT city_pkey PRIMARY KEY (id); + +ALTER TABLE ONLY country + ADD CONSTRAINT country_pkey PRIMARY KEY (code); + +ALTER TABLE ONLY countrylanguage + ADD CONSTRAINT countrylanguage_pkey PRIMARY KEY (countrycode, "language"); + +-- CTE1(inlined) in CTE2(no-inlined) case +explain verbose with country as +(select country.code,country.name COUNTRY, city.name CAPITAL, language, isofficial, percentage + FROM country,city,countrylanguage + WHERE country.code = countrylanguage.countrycode + and country.capital = city.id + and country.continent = 'Europe'), +countrylanguage as +(select country.code,country.COUNTRY,country.language,country.isofficial,country.percentage + FROM country,countrylanguage + WHERE country.code = countrylanguage.countrycode +) +select * from +(select * from country where isofficial='True') country, +(select * from countrylanguage where percentage > 50) countrylanguage +where country.percentage = countrylanguage.percentage order by countrylanguage.COUNTRY,country.language LIMIT 40; + +-- CTE in the main query and subqueries within the main query +explain verbose with bad_headofstates as +( + select country.code,country.name,country.headofstate,countrylanguage.language + from + country,countrylanguage + where country.code = countrylanguage.countrycode and countrylanguage.isofficial=true + and (country.gnp < country.gnpold or country.gnp < 3000) +) +select OUTERMOST_FOO.*,bad_headofstates.headofstate from ( +select avg(population),region from +( +select FOO.*,bad_headofstates.headofstate,city.name +from +(select bad_headofstates.code,country.capital,country.region,country.population from +bad_headofstates,country where bad_headofstates.code = country.code) FOO, bad_headofstates,city +where FOO.code = bad_headofstates.code and FOO.capital = city.id) OUTER_FOO +group by region ) OUTERMOST_FOO,bad_headofstates,country +where country.code = bad_headofstates.code and country.region = OUTERMOST_FOO.region +order by OUTERMOST_FOO.region,bad_headofstates.headofstate LIMIT 40; + +-- start_ignore +drop table city; +drop table country; +drop table countrylanguage; +-- end_ignore + +-- inlined CTEs +CREATE TABLE t5 AS SELECT i as c, i+1 as d from generate_series(1,10)i; +CREATE TABLE t6 AS SELECT i as a, i+1 as b from generate_series(1,10)i; + +-- inlined CTEs should have not unused columns(ex. t5.*, t6.* in output) +explain verbose WITH w AS (SELECT a, b from t6 where b < 5) +SELECT * +FROM t6, + (WITH v AS (SELECT c, d FROM t5, w WHERE c = w.a AND c < 2) + SELECT v1.c, v1.d FROM v v1, v v2 WHERE v1.c = v2.c AND v1.d > 1 + ) x +WHERE t6.a = x.c ORDER BY 1; + +WITH w AS (SELECT a, b from t6 where b < 5) +SELECT * +FROM t6, + (WITH v AS (SELECT c, d FROM t5, w WHERE c = w.a AND c < 2) + SELECT v1.c, v1.d FROM v v1, v v2 WHERE v1.c = v2.c AND v1.d > 1 + ) x +WHERE t6.a = x.c ORDER BY 1; + +CREATE TABLE t7 (f1 integer, f2 integer, f3 float); + +INSERT INTO t7 VALUES (1, 2, 3); +INSERT INTO t7 VALUES (2, 3, 4); +INSERT INTO t7 VALUES (3, 4, 5); +INSERT INTO t7 VALUES (1, 1, 1); +INSERT INTO t7 VALUES (2, 2, 2); +INSERT INTO t7 VALUES (3, 3, 3); +INSERT INTO t7 VALUES (6, 7, 8); +INSERT INTO t7 VALUES (8, 9, NULL); + +-- inlined CTEs should used the origin cexpression with recreated +explain (verbose, costs off) +with x as (select * from (select f1 from t7) ss) +select * from x where f1 = 1; + +with x as (select * from (select f1 from t7) ss) +select * from x where f1 = 1; + +-- start_ignore +drop table t5; +drop table t6; +drop table t7; + +drop schema cte_prune cascade; +-- end_ignore \ No newline at end of file From 2ced9eaa9afae2b09f05f8487be0acf9fa264e42 Mon Sep 17 00:00:00 2001 From: Yao Wang Date: Thu, 24 Aug 2023 15:36:41 +0800 Subject: [PATCH 006/244] Add explain analyze detailed info of hash agg (#15917) After upgrading to GPDB7, we lost the detailed cdb executor instruments of hashagg since the code base has been changed in a large size. The patch is to re-implement explain analyze related code of hashagg to provide more critical info for troubleshooting issues. --- src/backend/commands/explain.c | 2 +- src/backend/executor/nodeAgg.c | 145 ++++++++++++++++++ src/include/executor/nodeAgg.h | 23 +++ src/include/lib/simplehash.h | 81 ++++++++++ src/test/regress/expected/explain_format.out | 54 ++++++- .../expected/explain_format_optimizer.out | 52 ++++++- src/test/regress/sql/explain_format.sql | 35 ++++- 7 files changed, 388 insertions(+), 4 deletions(-) diff --git a/src/backend/commands/explain.c b/src/backend/commands/explain.c index bfd5f98219c..c89d088c4a2 100644 --- a/src/backend/commands/explain.c +++ b/src/backend/commands/explain.c @@ -4266,7 +4266,7 @@ show_hashagg_info(AggState *aggstate, ExplainState *es) } /* Display stats for each parallel worker */ - if (es->analyze && aggstate->shared_info != NULL) + if (aggstate->shared_info != NULL) { for (int n = 0; n < aggstate->shared_info->num_workers; n++) { diff --git a/src/backend/executor/nodeAgg.c b/src/backend/executor/nodeAgg.c index 881d09e5375..1c49e426d2d 100644 --- a/src/backend/executor/nodeAgg.c +++ b/src/backend/executor/nodeAgg.c @@ -481,6 +481,8 @@ static void build_pertrans_for_aggref(AggStatePerTrans pertrans, static void ExecEagerFreeAgg(AggState *node); +void agg_hash_explain_extra_message(AggState *aggstate); + /* * Select the current grouping set; affects current_set and * curaggcontext. @@ -1540,6 +1542,16 @@ build_hash_tables(AggState *aggstate) memory); build_hash_table(aggstate, setno, nbuckets); + + /* initialize some statistic info of hash table */ + perhash->num_output_groups = 0; + perhash->num_spill_parts = 0; + perhash->num_expansions = 0; + perhash->bucket_total = 0; + perhash->bucket_used = 0; + perhash->chain_count = 0; + perhash->chain_length_total = 0; + perhash->chain_length_max = 0; } aggstate->hash_ngroups_current = 0; @@ -1966,6 +1978,7 @@ hash_agg_enter_spill_mode(AggState *aggstate) hashagg_spill_init(aggstate, spill, aggstate->hash_tapeinfo, 0, perhash->aggnode->numGroups, aggstate->hashentrysize); + perhash->num_spill_parts += spill->npartitions; } if (aggstate->ss.ps.instrument) @@ -2019,6 +2032,12 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions) } /* update hashentrysize estimate based on contents */ + /* + * Greenplum doesn't use hashentrysize in the instrumentation, it will + * calculate hash table chain length to get an accurate number. + * + * See the following code to collect hash table statistic info. + */ if (aggstate->hash_ngroups_current > 0) { aggstate->hashentrysize = @@ -2031,6 +2050,47 @@ hash_agg_update_metrics(AggState *aggstate, bool from_tape, int npartitions) Instrumentation *instrument = aggstate->ss.ps.instrument; instrument->workmemused = aggstate->hash_mem_peak; + + /* + * workmemwanted to avoid scratch i/o, that how much memory is needed + * if we want to load into the hashtable at once: + * + * 1. add meta_mem only when from_tape is false, because when we are + * reading from tape/spilled file, we can reuse the existing hash + * table's meta. + * 2. add hash_mem every time. + * 3. don't add buffer_mem since it's unnecessary when we can load into + * into the memory at once. + */ + if (!from_tape) + instrument->workmemwanted += meta_mem; + instrument->workmemwanted += hashkey_mem; + + /* Scan all perhashs and collect hash table statistic info */ + for (int setno = 0; setno < aggstate->num_hashes; setno++) + { + AggStatePerHash perhash = &aggstate->perhash[setno]; + tuplehash_hash *hashtab = perhash->hashtable->hashtab; + + Assert(hashtab); + + perhash->num_expansions += hashtab->num_expansions; + perhash->bucket_total += hashtab->size; + perhash->bucket_used += hashtab->members; + if (hashtab->members > 0) + { + uint32 perht_chain_length_total = 0; + uint32 perht_chain_count = 0; + + /* collect statistic info of chain length per hash table */ + tuplehash_coll_stat(hashtab, + &(perhash->chain_length_max), + &perht_chain_length_total, + &perht_chain_count); + perhash->chain_count += perht_chain_count; + perhash->chain_length_total += perht_chain_length_total; + } + } } } @@ -2225,6 +2285,7 @@ lookup_hash_entries(AggState *aggstate) hashagg_spill_init(aggstate, spill, aggstate->hash_tapeinfo, 0, perhash->aggnode->numGroups, aggstate->hashentrysize); + perhash->num_spill_parts += spill->npartitions; hashagg_spill_tuple(aggstate, spill, slot, hash); pergroup[setno] = NULL; @@ -2277,6 +2338,13 @@ ExecAgg(PlanState *pstate) return result; } + /* Save statistics into the cdbexplainbuf for EXPLAIN ANALYZE */ + if (node->ss.ps.instrument && + (node->ss.ps.instrument)->need_cdb && + (node->phase->aggstrategy == AGG_HASHED || + node->phase->aggstrategy == AGG_MIXED)) + agg_hash_explain_extra_message(node); + return NULL; } @@ -2807,6 +2875,7 @@ agg_refill_hash_table(AggState *aggstate) spill_initialized = true; hashagg_spill_init(aggstate, &spill, tapeinfo, batch->used_bits, batch->input_card, aggstate->hashentrysize); + aggstate->perhash[aggstate->current_set].num_spill_parts += spill.npartitions; } /* no memory for a new group, spill */ hashagg_spill_tuple(aggstate, &spill, spillslot, hash); @@ -2981,6 +3050,8 @@ agg_retrieve_hash_table_in_memory(AggState *aggstate) } } + perhash->num_output_groups++; + /* * Clear the per-output-tuple context for each group * @@ -5139,3 +5210,77 @@ ReuseHashTable(AggState *node) !node->streaming && !bms_overlap(node->ss.ps.chgParam, aggnode->aggParams)); } + +/* + * Save statistics into the cdbexplainbuf for EXPLAIN ANALYZE + */ +void +agg_hash_explain_extra_message(AggState *aggstate) +{ + /* + * Check cdbexplain_depositStatsToNode(), Greenplum only saves extra + * message text for the most interesting winning qExecs. + */ + StringInfo hbuf = aggstate->ss.ps.cdbexplainbuf; + uint64 sum_num_expansions = 0; + uint64 sum_output_groups = 0; + uint64 sum_spill_parts = 0; + uint64 sum_chain_length_total = 0; + uint64 sum_chain_count = 0; + uint32 chain_length_max = 0; + uint64 sum_bucket_used = 0; + uint64 sum_bucket_total = 0; + + Assert(hbuf); + + appendStringInfo(hbuf, "hash table(s): %d", aggstate->num_hashes); + + /* Scan all perhashs and collect statistic info */ + for (int setno = 0; setno < aggstate->num_hashes; setno++) + { + AggStatePerHash perhash = &aggstate->perhash[setno]; + + /* spill statistic info */ + if (aggstate->hash_ever_spilled) + { + sum_output_groups += perhash->num_output_groups; + sum_spill_parts += perhash->num_spill_parts; + } + + /* inner hash table statistic info */ + if (perhash->chain_count > 0) + { + sum_chain_length_total += perhash->chain_length_total; + sum_chain_count += perhash->chain_count; + if (perhash->chain_length_max > chain_length_max) + chain_length_max = perhash->chain_length_max; + sum_bucket_used = perhash->bucket_used; + sum_bucket_total = perhash->bucket_total; + sum_num_expansions += perhash->num_expansions; + } + } + + if (aggstate->hash_ever_spilled) + { + appendStringInfo(hbuf, + "; " UINT64_FORMAT " groups total in %d batches, " UINT64_FORMAT + " spill partitions; disk usage: " INT64_FORMAT "KB", + sum_output_groups, + aggstate->hash_batches_used, + sum_spill_parts, + aggstate->hash_disk_used); + } + + if (sum_chain_count > 0) + { + appendStringInfo(hbuf, + "; chain length %.1f avg, %d max;" + " using " INT64_FORMAT " of " INT64_FORMAT " buckets;" + " total " INT64_FORMAT " expansions.\n", + (double)sum_chain_length_total / sum_chain_count, + chain_length_max, + sum_bucket_used, + sum_bucket_total, + sum_num_expansions); + } +} diff --git a/src/include/executor/nodeAgg.h b/src/include/executor/nodeAgg.h index 60fd9f960f8..5e4e21f250a 100644 --- a/src/include/executor/nodeAgg.h +++ b/src/include/executor/nodeAgg.h @@ -311,6 +311,29 @@ typedef struct AggStatePerHashData AttrNumber *hashGrpColIdxInput; /* hash col indices in input slot */ AttrNumber *hashGrpColIdxHash; /* indices in hash table tuples */ Agg *aggnode; /* original Agg node, for numGroups etc. */ + + /* + * Some statistic info of hash table, used for EXPLAIN ANALYZE. + * Note that they are accumulated info and will not be reset even + * after the hash table is reset. + */ + + /* number of groups/entries output by the iterator */ + uint64 num_output_groups; + /* number of spilled partitions */ + uint64 num_spill_parts; + /* number of hash table expansions */ + uint32 num_expansions; + /* total number of buckets */ + uint64 bucket_total; + /* number of used buckets */ + uint64 bucket_used; + /* number of all chains */ + uint64 chain_count; + /* total length of all chains */ + uint64 chain_length_total; + /* max chain length */ + uint32 chain_length_max; } AggStatePerHashData; diff --git a/src/include/lib/simplehash.h b/src/include/lib/simplehash.h index a4a25c57f21..8b4af795544 100644 --- a/src/include/lib/simplehash.h +++ b/src/include/lib/simplehash.h @@ -126,6 +126,7 @@ #define SH_ALLOCATE SH_MAKE_NAME(allocate) #define SH_FREE SH_MAKE_NAME(free) #define SH_STAT SH_MAKE_NAME(stat) +#define SH_COLL_STAT SH_MAKE_NAME(coll_stat) /* internal helper functions (no externally visible prototypes) */ #define SH_COMPUTE_PARAMETERS SH_MAKE_NAME(compute_parameters) @@ -169,6 +170,14 @@ typedef struct SH_TYPE /* user defined data, useful for callbacks */ void *private_data; + + /* + * number of times hash table is expanded + * + * Since the max size of hash table is UINT32_MAX, uint32 is good + * enough for the number of expanded times. + */ + uint32 num_expansions; } SH_TYPE; typedef enum SH_STATUS @@ -244,6 +253,12 @@ SH_SCOPE SH_ELEMENT_TYPE *SH_ITERATE(SH_TYPE * tb, SH_ITERATOR * iter); /* void _stat(_hash *tb */ SH_SCOPE void SH_STAT(SH_TYPE * tb); +SH_SCOPE void +SH_COLL_STAT(SH_TYPE * tb, + uint32 * max_chain_length, + uint32 * total_chain_length, + uint32 * chain_count); + #endif /* SH_DECLARE */ @@ -450,6 +465,7 @@ SH_CREATE(MemoryContext ctx, uint32 nelements, void *private_data) SH_COMPUTE_PARAMETERS(tb, size); tb->data = SH_ALLOCATE(tb, sizeof(SH_ELEMENT_TYPE) * tb->size); + tb->num_expansions = 0; return tb; } @@ -468,6 +484,7 @@ SH_RESET(SH_TYPE * tb) { memset(tb->data, 0, sizeof(SH_ELEMENT_TYPE) * tb->size); tb->members = 0; + tb->num_expansions = 0; } /* @@ -580,6 +597,7 @@ SH_GROW(SH_TYPE * tb, uint64 newsize) } } + (tb->num_expansions)++; SH_FREE(tb, olddata); } @@ -1120,6 +1138,69 @@ SH_STAT(SH_TYPE * tb) total_collisions, max_collisions, avg_collisions); } +/* + * Greenplum specific + * + * Collect some statistics about the state of the hashtable. Major code was + * copied from SH_STAT() with some modifications to keep consistent with GPDB6. + */ +SH_SCOPE void +SH_COLL_STAT(SH_TYPE * tb, + uint32 * max_chain_length, + uint32 * total_chain_length, + uint32 * chain_count) +{ + *total_chain_length = 0; + *chain_count = 0; + uint32 last_dist = 0; + + for (int i = 0; i < tb->size; i++) + { + uint32 hash; + uint32 optimal; + uint32 dist; + SH_ELEMENT_TYPE *elem; + + elem = &tb->data[i]; + + if (elem->status != SH_STATUS_IN_USE) + continue; + + hash = SH_ENTRY_HASH(tb, elem); + optimal = SH_INITIAL_BUCKET(tb, hash); + dist = SH_DISTANCE_FROM_OPTIMAL(tb, optimal, i); + + /* + * Different from SH_STAT(), always calculate chain length from 1 but + * not 0, e.g. when there is only one element in bucket, the length + * is 1. + */ + dist++; + + /* + * In same chain, dist must be always increasing. If dist < last_dist, + * we must hit a new chain; take the length of old chain into account. + */ + if (dist < last_dist) + { + if (last_dist > *max_chain_length) + *max_chain_length = last_dist; + *total_chain_length += last_dist; + (*chain_count)++; + } + last_dist = dist; + } + + /* Count the last chain. */ + if (last_dist != 0) + { + if (last_dist > *max_chain_length) + *max_chain_length = last_dist; + *total_chain_length += last_dist; + (*chain_count)++; + } +} + #endif /* SH_DEFINE */ diff --git a/src/test/regress/expected/explain_format.out b/src/test/regress/expected/explain_format.out index 1675c8a2153..c8c452cc781 100644 --- a/src/test/regress/expected/explain_format.out +++ b/src/test/regress/expected/explain_format.out @@ -17,6 +17,8 @@ -- s/Memory used: \d+\w?B/Memory used: ###B/ -- m/Memory Usage: \d+\w?B/ -- s/Memory Usage: \d+\w?B/Memory Usage: ###B/ +-- m/Memory wanted: \d+\w?kB/ +-- s/Memory wanted: \d+\w?kB/Memory wanted: ###kB/ -- m/Peak Memory Usage: \d+/ -- s/Peak Memory Usage: \d+/Peak Memory Usage: ###/ -- m/Buckets: \d+/ @@ -583,9 +585,59 @@ QUERY PLAN } ] (1 row) --- explain_processing_on +-- start_matchsubs +-- m/Extra Text: \(seg\d+\) hash table\(s\): \d+; \d+ groups total in \d+ batches, \d+ spill partitions; disk usage: \d+KB; chain length \d+.\d+ avg, \d+ max; using \d+ of \d+ buckets; total \d+ expansions./ +-- s/Extra Text: \(seg\d+\) hash table\(s\): \d+; \d+ groups total in \d+ batches, \d+ spill partitions; disk usage: \d+KB; chain length \d+.\d+ avg, \d+ max; using \d+ of \d+ buckets; total \d+ expansions./Extra Text: (seg0) hash table(s): ###; ### groups total in ### batches, ### spill partitions; disk usage: ###KB; chain length ###.## avg, ### max; using ## of ### buckets; total ### expansions./ +-- m/Work_mem: \d+K bytes max, \d+K bytes wanted/ +-- s/Work_mem: \d+K bytes max, \d+K bytes wanted/Work_mem: ###K bytes max, ###K bytes wanted/ +-- end_matchsubs +-- Greenplum hash table extra message +CREATE TABLE test_src_tbl AS +SELECT i % 10000 AS a, i % 10000 + 1 AS b FROM generate_series(1, 50000) i DISTRIBUTED BY (a); +ANALYZE test_src_tbl; +-- Enable optimizer_enable_hashagg, and set statement_mem to a small value to force spilling +set optimizer_enable_hashagg = on; +SET statement_mem = '1000kB'; +-- Hashagg with spilling +CREATE TABLE test_hashagg_spill AS +SELECT a, COUNT(DISTINCT b) AS b FROM test_src_tbl GROUP BY a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +EXPLAIN (analyze, costs off) SELECT a, COUNT(DISTINCT b) AS b FROM test_src_tbl GROUP BY a; +QUERY PLAN +Gather Motion 3:1 (slice1; segments: 3) (actual time=8.948..21.656 rows=10000 loops=1) + -> HashAggregate (actual time=13.950..15.183 rows=3386 loops=1) + Group Key: a + Extra Text: (seg0) hash table(s): 1; 3368 groups total in 4 batches, 2576 spill partitions; disk usage: 1024KB; chain length 2.9 avg, 9 max; using 3368 of 20480 buckets; total 0 expansions. + + -> HashAggregate (actual time=12.113..12.658 rows=3386 loops=1) + Group Key: a, b + Extra Text: (seg0) hash table(s): 1; chain length 2.3 avg, 5 max; using 3368 of 8192 buckets; total 1 expansions. + + -> Seq Scan on test_src_tbl (actual time=0.015..1.748 rows=16930 loops=1) +Planning Time: 0.167 ms + (slice0) Executor memory: 236K bytes. +* (slice1) Executor memory: 657K bytes avg x 3 workers, 722K bytes max (seg0). Work_mem: 753K bytes max, 721K bytes wanted. +Memory used: 1000kB +Memory wanted: 1640kB +Optimizer: Postgres query optimizer +Execution Time: 22.346 ms +(17 rows) +-- Hashagg with grouping sets +CREATE TABLE test_hashagg_groupingsets AS +SELECT a, avg(b) AS b FROM test_src_tbl GROUP BY grouping sets ((a), (b)); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Greenplum Database data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +-- The planner generates multiple hash tables but ORCA uses Shared Scan. +-- flaky test +-- EXPLAIN (analyze, costs off) SELECT a, avg(b) AS b FROM test_src_tbl GROUP BY grouping sets ((a), (b)); +RESET optimizer_enable_hashagg; +RESET statement_mem; -- Cleanup DROP TABLE boxes; DROP TABLE apples; DROP TABLE box_locations; DROP TABLE jsonexplaintest; +DROP TABLE test_src_tbl; +DROP TABLE test_hashagg_spill; +DROP TABLE test_hashagg_groupingsets; diff --git a/src/test/regress/expected/explain_format_optimizer.out b/src/test/regress/expected/explain_format_optimizer.out index 74a0d03e09d..459a466c993 100644 --- a/src/test/regress/expected/explain_format_optimizer.out +++ b/src/test/regress/expected/explain_format_optimizer.out @@ -17,6 +17,8 @@ -- s/Memory used: \d+\w?B/Memory used: ###B/ -- m/Memory Usage: \d+\w?B/ -- s/Memory Usage: \d+\w?B/Memory Usage: ###B/ +-- m/Memory wanted: \d+\w?kB/ +-- s/Memory wanted: \d+\w?kB/Memory wanted: ###kB/ -- m/Peak Memory Usage: \d+/ -- s/Peak Memory Usage: \d+/Peak Memory Usage: ###/ -- m/Buckets: \d+/ @@ -530,9 +532,57 @@ QUERY PLAN } ] (1 row) --- explain_processing_on +-- start_matchsubs +-- m/Extra Text: \(seg\d+\) hash table\(s\): \d+; \d+ groups total in \d+ batches, \d+ spill partitions; disk usage: \d+KB; chain length \d+.\d+ avg, \d+ max; using \d+ of \d+ buckets; total \d+ expansions./ +-- s/Extra Text: \(seg\d+\) hash table\(s\): \d+; \d+ groups total in \d+ batches, \d+ spill partitions; disk usage: \d+KB; chain length \d+.\d+ avg, \d+ max; using \d+ of \d+ buckets; total \d+ expansions./Extra Text: (seg0) hash table(s): ###; ### groups total in ### batches, ### spill partitions; disk usage: ###KB; chain length ###.## avg, ### max; using ## of ### buckets; total ### expansions./ +-- m/Work_mem: \d+K bytes max, \d+K bytes wanted/ +-- s/Work_mem: \d+K bytes max, \d+K bytes wanted/Work_mem: ###K bytes max, ###K bytes wanted/ +-- end_matchsubs +-- Greenplum hash table extra message +CREATE TABLE test_src_tbl AS +SELECT i % 10000 AS a, i % 10000 + 1 AS b FROM generate_series(1, 50000) i DISTRIBUTED BY (a); +ANALYZE test_src_tbl; +-- Enable optimizer_enable_hashagg, and set statement_mem to a small value to force spilling +set optimizer_enable_hashagg = on; +SET statement_mem = '1000kB'; +-- Hashagg with spilling +CREATE TABLE test_hashagg_spill AS +SELECT a, COUNT(DISTINCT b) AS b FROM test_src_tbl GROUP BY a; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. +EXPLAIN (analyze, costs off) SELECT a, COUNT(DISTINCT b) AS b FROM test_src_tbl GROUP BY a; +QUERY PLAN +Gather Motion 3:1 (slice1; segments: 3) (actual time=9.551..21.820 rows=10000 loops=1) + -> HashAggregate (actual time=9.028..10.280 rows=3386 loops=1) + Group Key: a + Extra Text: (seg0) hash table(s): 1; 3368 groups total in 4 batches, 2576 spill partitions; disk usage: 1024KB; chain length 2.9 avg, 9 max; using 3368 of 20480 buckets; total 0 expansions. + + -> HashAggregate (actual time=7.066..7.628 rows=3386 loops=1) + Group Key: a, b + Extra Text: (seg0) hash table(s): 1; 3368 groups total in 4 batches, 1264 spill partitions; disk usage: 1024KB; chain length 2.3 avg, 5 max; using 3368 of 40960 buckets; total 1 expansions. + + -> Seq Scan on test_src_tbl (actual time=0.020..1.349 rows=16930 loops=1) +Planning Time: 5.518 ms + (slice0) Executor memory: 236K bytes. +* (slice1) Executor memory: 709K bytes avg x 3 workers, 876K bytes max (seg0). Work_mem: 753K bytes max, 721K bytes wanted. +Memory used: 1000kB +Memory wanted: 1640kB +Optimizer: GPORCA +Execution Time: 22.843 ms +(17 rows) +-- Hashagg with grouping sets +CREATE TABLE test_hashagg_groupingsets AS +SELECT a, avg(b) AS b FROM test_src_tbl GROUP BY grouping sets ((a), (b)); +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause. Creating a NULL policy entry. +-- The planner generates multiple hash tables but ORCA uses Shared Scan. +-- flaky test +-- EXPLAIN (analyze, costs off) SELECT a, avg(b) AS b FROM test_src_tbl GROUP BY grouping sets ((a), (b)); +RESET optimizer_enable_hashagg; +RESET statement_mem; -- Cleanup DROP TABLE boxes; DROP TABLE apples; DROP TABLE box_locations; DROP TABLE jsonexplaintest; +DROP TABLE test_src_tbl; +DROP TABLE test_hashagg_spill; +DROP TABLE test_hashagg_groupingsets; diff --git a/src/test/regress/sql/explain_format.sql b/src/test/regress/sql/explain_format.sql index 602891698fd..e04cba90947 100644 --- a/src/test/regress/sql/explain_format.sql +++ b/src/test/regress/sql/explain_format.sql @@ -17,6 +17,8 @@ -- s/Memory used: \d+\w?B/Memory used: ###B/ -- m/Memory Usage: \d+\w?B/ -- s/Memory Usage: \d+\w?B/Memory Usage: ###B/ +-- m/Memory wanted: \d+\w?kB/ +-- s/Memory wanted: \d+\w?kB/Memory wanted: ###kB/ -- m/Peak Memory Usage: \d+/ -- s/Peak Memory Usage: \d+/Peak Memory Usage: ###/ -- m/Buckets: \d+/ @@ -110,10 +112,41 @@ EXPLAIN (FORMAT XML, COSTS OFF) SELECT * FROM generate_series(1, 10); CREATE TABLE jsonexplaintest (i int4) PARTITION BY RANGE (i) (START(1) END(3) EVERY(1)); EXPLAIN (FORMAT JSON, COSTS OFF) SELECT * FROM jsonexplaintest WHERE i = 2; --- explain_processing_on +-- start_matchsubs +-- m/Extra Text: \(seg\d+\) hash table\(s\): \d+; \d+ groups total in \d+ batches, \d+ spill partitions; disk usage: \d+KB; chain length \d+.\d+ avg, \d+ max; using \d+ of \d+ buckets; total \d+ expansions./ +-- s/Extra Text: \(seg\d+\) hash table\(s\): \d+; \d+ groups total in \d+ batches, \d+ spill partitions; disk usage: \d+KB; chain length \d+.\d+ avg, \d+ max; using \d+ of \d+ buckets; total \d+ expansions./Extra Text: (seg0) hash table(s): ###; ### groups total in ### batches, ### spill partitions; disk usage: ###KB; chain length ###.## avg, ### max; using ## of ### buckets; total ### expansions./ +-- m/Work_mem: \d+K bytes max, \d+K bytes wanted/ +-- s/Work_mem: \d+K bytes max, \d+K bytes wanted/Work_mem: ###K bytes max, ###K bytes wanted/ +-- end_matchsubs +-- Greenplum hash table extra message +CREATE TABLE test_src_tbl AS +SELECT i % 10000 AS a, i % 10000 + 1 AS b FROM generate_series(1, 50000) i DISTRIBUTED BY (a); +ANALYZE test_src_tbl; + +-- Enable optimizer_enable_hashagg, and set statement_mem to a small value to force spilling +set optimizer_enable_hashagg = on; +SET statement_mem = '1000kB'; + +-- Hashagg with spilling +CREATE TABLE test_hashagg_spill AS +SELECT a, COUNT(DISTINCT b) AS b FROM test_src_tbl GROUP BY a; +EXPLAIN (analyze, costs off) SELECT a, COUNT(DISTINCT b) AS b FROM test_src_tbl GROUP BY a; + +-- Hashagg with grouping sets +CREATE TABLE test_hashagg_groupingsets AS +SELECT a, avg(b) AS b FROM test_src_tbl GROUP BY grouping sets ((a), (b)); +-- The planner generates multiple hash tables but ORCA uses Shared Scan. +-- flaky test +-- EXPLAIN (analyze, costs off) SELECT a, avg(b) AS b FROM test_src_tbl GROUP BY grouping sets ((a), (b)); + +RESET optimizer_enable_hashagg; +RESET statement_mem; -- Cleanup DROP TABLE boxes; DROP TABLE apples; DROP TABLE box_locations; DROP TABLE jsonexplaintest; +DROP TABLE test_src_tbl; +DROP TABLE test_hashagg_spill; +DROP TABLE test_hashagg_groupingsets; From 65e8a65a3f8d59968303879926da3fd0a317cefe Mon Sep 17 00:00:00 2001 From: "Jianghua.yjh" Date: Tue, 17 Jun 2025 07:59:43 -0700 Subject: [PATCH 007/244] Fix potential use-after-free of viewQuery in ExecRefreshMatView (#1159) In ExecRefreshMatView(), the pointer viewQuery may be invalidated when make_new_heap_with_colname() closes the old heap via table_close(OldHeap, NoLock). This happens because table_close() can release relcache entries and associated rules, including rd_rules, which stores the viewQuery. To avoid accessing a possibly freed memory region, make a deep copy of viewQuery before invoking make_new_heap_with_colname(), and use this copy as dataQuery for further processing. Fix: ERROR: unrecognized node type: 2139062143 (copyfuncs.c:7663) Author: Jianghua Yang --- src/backend/commands/matview.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/backend/commands/matview.c b/src/backend/commands/matview.c index e56e9d76fb9..ea3408654c2 100644 --- a/src/backend/commands/matview.c +++ b/src/backend/commands/matview.c @@ -463,7 +463,8 @@ ExecRefreshMatView(RefreshMatViewStmt *stmt, const char *queryString, if (!stmt->skipData && RelationIsIVM(matviewRel)) dataQuery = rewriteQueryForIMMV(viewQuery,NIL); else - dataQuery = viewQuery; + /* viewQuery maybe released in make_new_heap_with_colname. */ + dataQuery = copyObject(viewQuery); /* * Check that there is a unique index with no WHERE clause on one or more From 5a6322dbc177583f0bb8fe2f0030b7812549fcc0 Mon Sep 17 00:00:00 2001 From: Smyatkin Maxim Date: Tue, 17 Jun 2025 19:30:58 +0300 Subject: [PATCH 008/244] Fix segfault when recovering 2pc transaction (#1078) * Fix segfault when recovering 2pc transaction When promoting a mirror segment due to failover we have seen a stacktrace like this: ``` "FATAL","58P01","requested WAL segment pg_xlog/00000023000071D50000001F has already been removed",,,,,,,0,,"xlogutils.c",580,"Stack trace: 1 0x557bdb9f09b6 postgres errstart + 0x236 2 0x557bdb5fc6cf postgres + 0xdb5fc6cf 3 0x557bdb5fd021 postgres read_local_xlog_page + 0x191 4 0x557bdb5fb922 postgres + 0xdb5fb922 5 0x557bdb5fba11 postgres XLogReadRecord + 0xa1 6 0x557bdb5e7767 postgres RecoverPreparedTransactions + 0xd7 7 0x557bdb5f608b postgres StartupXLOG + 0x2a3b 8 0x557bdb870a89 postgres StartupProcessMain + 0x139 9 0x557bdb62f489 postgres AuxiliaryProcessMain + 0x549 10 0x557bdb86d275 postgres + 0xdb86d275 11 0x557bdb8704e3 postgres PostmasterMain + 0x1213 12 0x557bdb56a1f7 postgres main + 0x497 13 0x7fded4a61c87 libc.so.6 __libc_start_main + 0xe7 ``` Note: stacktrace is from one of our production GP clusters and might be slightly different from what we will see in cloudberry, but the failure is still present here as well. Testcase proves it. PG13 and PG14 have a fix for this bug, but it's doesn't have any test case and looks like we didn't cherry-pick that far. The discussion can be found here: https://www.postgresql.org/message-id/flat/743b9b45a2d4013bd90b6a5cba8d6faeb717ee34.camel%40cybertec.at In a few words, StartupXLOG() renames the last wal segment to .partial but tries to read it by the old name later in RecoverPreparedTransactions(). The fix is mostly borrowed from PG14 postgres/postgres@f663b00 with some cloudberry-related exceptions. Also added a regression test which segfaults without this fix for any version of GP, PG<=12 or Cloudberry. * Add stable ordering to testcase --------- Co-authored-by: Jianghua.yjh --- src/backend/access/transam/xlog.c | 146 +++++++-------- .../startup_rename_prepared_xlog.out | 173 ++++++++++++++++++ src/test/isolation2/isolation2_schedule | 1 + .../startup_rename_prepared_xlog.sql | 89 +++++++++ 4 files changed, 336 insertions(+), 73 deletions(-) create mode 100644 src/test/isolation2/expected/segwalrep/startup_rename_prepared_xlog.out create mode 100644 src/test/isolation2/sql/segwalrep/startup_rename_prepared_xlog.sql diff --git a/src/backend/access/transam/xlog.c b/src/backend/access/transam/xlog.c index 722bea2040b..3fb9f121b93 100644 --- a/src/backend/access/transam/xlog.c +++ b/src/backend/access/transam/xlog.c @@ -8321,6 +8321,79 @@ StartupXLOG(void) CreateCheckPoint(CHECKPOINT_END_OF_RECOVERY | CHECKPOINT_IMMEDIATE); } + /* + * Preallocate additional log files, if wanted. + */ + PreallocXlogFiles(EndOfLog); + + /* + * Okay, we're officially UP. + */ + InRecovery = false; + + /* + * Hook for plugins to do additional startup works. + * + * Allow to write any WALs in hook. + */ + if (Startup_hook) + { + LocalSetXLogInsertAllowed(); + (*Startup_hook) (); + LocalXLogInsertAllowed = -1; + } + + /* start the archive_timeout timer and LSN running */ + XLogCtl->lastSegSwitchTime = (pg_time_t) time(NULL); + XLogCtl->lastSegSwitchLSN = EndOfLog; + + /* also initialize latestCompletedXid, to nextXid - 1 */ + LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); + ShmemVariableCache->latestCompletedXid = ShmemVariableCache->nextXid; + ShmemVariableCache->latestCompletedGxid = ShmemVariableCache->nextGxid; + FullTransactionIdRetreat(&ShmemVariableCache->latestCompletedXid); + if (IsNormalProcessingMode()) + elog(LOG, "latest completed transaction id is %u and next transaction id is %u", + XidFromFullTransactionId(ShmemVariableCache->latestCompletedXid), + XidFromFullTransactionId(ShmemVariableCache->nextXid)); + LWLockRelease(ProcArrayLock); + + /* + * Start up subtrans, if not already done for hot standby. (commit + * timestamps are started below, if necessary.) + */ + if (standbyState == STANDBY_DISABLED) + { + StartupCLOG(); + StartupSUBTRANS(oldestActiveXID); + DistributedLog_Startup(oldestActiveXID, + XidFromFullTransactionId(ShmemVariableCache->nextXid)); + } + + /* + * Perform end of recovery actions for any SLRUs that need it. + */ + TrimCLOG(); + TrimMultiXact(); + + /* Reload shared-memory state for prepared transactions */ + RecoverPreparedTransactions(); + + /* Shut down xlogreader */ + if (readFile >= 0) + { + close(readFile); + readFile = -1; + } + XLogReaderFree(xlogreader); + + /* + * If any of the critical GUCs have changed, log them before we allow + * backends to write WAL. + */ + LocalSetXLogInsertAllowed(); + XLogReportParameters(); + if (ArchiveRecoveryRequested) { /* @@ -8402,28 +8475,6 @@ StartupXLOG(void) } } - /* - * Preallocate additional log files, if wanted. - */ - PreallocXlogFiles(EndOfLog); - - /* - * Okay, we're officially UP. - */ - InRecovery = false; - - /* - * Hook for plugins to do additional startup works. - * - * Allow to write any WALs in hook. - */ - if (Startup_hook) - { - LocalSetXLogInsertAllowed(); - (*Startup_hook) (); - LocalXLogInsertAllowed = -1; - } - /* * If we are a standby with contentid -1 and undergoing promotion, * update ourselves as the new master in catalog. This does not @@ -8433,60 +8484,9 @@ StartupXLOG(void) bool needToPromoteCatalog = (IS_QUERY_DISPATCHER() && ControlFile->state == DB_IN_ARCHIVE_RECOVERY); - /* start the archive_timeout timer and LSN running */ - XLogCtl->lastSegSwitchTime = (pg_time_t) time(NULL); - XLogCtl->lastSegSwitchLSN = EndOfLog; - - /* also initialize latestCompletedXid, to nextXid - 1 */ - LWLockAcquire(ProcArrayLock, LW_EXCLUSIVE); - ShmemVariableCache->latestCompletedXid = ShmemVariableCache->nextXid; - ShmemVariableCache->latestCompletedGxid = ShmemVariableCache->nextGxid; - FullTransactionIdRetreat(&ShmemVariableCache->latestCompletedXid); - if (IsNormalProcessingMode()) - elog(LOG, "latest completed transaction id is %u and next transaction id is %u", - XidFromFullTransactionId(ShmemVariableCache->latestCompletedXid), - XidFromFullTransactionId(ShmemVariableCache->nextXid)); - LWLockRelease(ProcArrayLock); - - /* - * Start up subtrans, if not already done for hot standby. (commit - * timestamps are started below, if necessary.) - */ - if (standbyState == STANDBY_DISABLED) - { - StartupCLOG(); - StartupSUBTRANS(oldestActiveXID); - DistributedLog_Startup(oldestActiveXID, - XidFromFullTransactionId(ShmemVariableCache->nextXid)); - } - - /* - * Perform end of recovery actions for any SLRUs that need it. - */ - TrimCLOG(); - TrimMultiXact(); - - /* Reload shared-memory state for prepared transactions */ - RecoverPreparedTransactions(); - if(IsNormalProcessingMode()) ereport(LOG, (errmsg("database system is ready"))); - /* Shut down xlogreader */ - if (readFile >= 0) - { - close(readFile); - readFile = -1; - } - XLogReaderFree(xlogreader); - - /* - * If any of the critical GUCs have changed, log them before we allow - * backends to write WAL. - */ - LocalSetXLogInsertAllowed(); - XLogReportParameters(); - /* * Local WAL inserts enabled, so it's time to finish initialization of * commit timestamp. diff --git a/src/test/isolation2/expected/segwalrep/startup_rename_prepared_xlog.out b/src/test/isolation2/expected/segwalrep/startup_rename_prepared_xlog.out new file mode 100644 index 00000000000..d51241e6aa0 --- /dev/null +++ b/src/test/isolation2/expected/segwalrep/startup_rename_prepared_xlog.out @@ -0,0 +1,173 @@ +-- Test a bug where the last XLOG segment is renamed to .partial before prepared +-- transactions contained in it are recovered. This would result in segfault at +-- RecoverPreparedTransactions. The failure is well described in PG hackers: +-- https://www.postgresql.org/message-id/743b9b45a2d4013bd90b6a5cba8d6faeb717ee34.camel%40cybertec.at +-- However, in GP due to an older version of PG this error is masked by always caching +-- the last opened xlog segment in XLogRead(). To actually trigger the failure we need to +-- create unfinished prepared transactions in TWO different xlog segments. + +-- Allow extra time for mirror promotion to complete recovery to avoid +-- gprecoverseg BEGIN failures due to gang creation failure as some primaries +-- are not up. Setting these increase the number of retries in gang creation in +-- case segment is in recovery. Approximately we want to wait 30 seconds. +!\retcode gpconfig -c gp_gang_creation_retry_count -v 120 --skipvalidation --masteronly; +(exited with code 0) +!\retcode gpconfig -c gp_gang_creation_retry_timer -v 1000 --skipvalidation --masteronly; +(exited with code 0) + +-- The last xlog segment is renamed to .partial only when archive mode is on +!\retcode gpconfig -c archive_mode -v on; +(exited with code 0) +!\retcode gpconfig -c archive_command -v '/bin/true'; +(exited with code 0) +!\retcode gpstop -rai; +(exited with code 0) + +1: create extension if not exists gp_inject_fault; +CREATE +1: create table t_rename1 (a int); +CREATE +1: create table t_rename2 (a int); +CREATE + +-- generate an orphaned prepare transaction in first segment +1: select gp_inject_fault('dtm_broadcast_prepare', 'suspend', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +-- assume (2), (1) are on different segments and one tuple is on the first segment. +-- the test finally double-check that. +2&: insert into t_rename1 values(2),(1); +1: select gp_wait_until_triggered_fault('dtm_broadcast_prepare', 1, dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) + +-- start another xlog segment (will be renamed to .partial on failover) +GP_IGNORE:-- start_ignore +GP_IGNORE:0U: select pg_switch_xlog(); +GP_IGNORE: pg_switch_xlog +GP_IGNORE:---------------- +GP_IGNORE: 0/C0B05E0 +GP_IGNORE:(1 row) +GP_IGNORE:-- end_ignore + +-- inject another fault and prepare transaction in the new xlog segment +1: select gp_inject_fault('dtm_broadcast_prepare', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +1: select gp_inject_fault('dtm_broadcast_prepare', 'suspend', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +-- assume (4), (3) are on different segments and one tuple is on the first segment. +-- the test finally double-check that. +3&: insert into t_rename2 values(2),(1); +1: select gp_wait_until_triggered_fault('dtm_broadcast_prepare', 1, dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_wait_until_triggered_fault +------------------------------- + Success: +(1 row) + +-- shutdown primary and make sure the segment is down +-1U: select pg_ctl((SELECT datadir from gp_segment_configuration c where c.role='p' and c.content=0), 'stop', 'immediate'); + pg_ctl +-------- + OK +(1 row) +1: select gp_request_fts_probe_scan(); + gp_request_fts_probe_scan +--------------------------- + t +(1 row) +1: select role, preferred_role from gp_segment_configuration where content = 0 order by role; + role | preferred_role +------+---------------- + m | p + p | m +(2 rows) + +-- double confirm that promote succeeds. +-- also double confirm that +-- 1. tuples (2) and (1) are located on two segments (thus we are testing 2pc with prepared transaction). +-- 2. there are tuples on the first segment (we have been testing on the first segment). +1: insert into t_rename1 values(2),(1); +INSERT 2 +1: select gp_segment_id, * from t_rename1 order by a; + gp_segment_id | a +---------------+--- + 1 | 1 + 0 | 2 +(2 rows) + +1: select gp_inject_fault('dtm_broadcast_prepare', 'reset', dbid) from gp_segment_configuration where role = 'p' and content = -1; + gp_inject_fault +----------------- + Success: +(1 row) +2<: <... completed> +INSERT 2 +3<: <... completed> +INSERT 2 + +-- confirm the "orphaned" prepared trnasaction commits finally. +1: select * from t_rename1 order by a; + a +--- + 1 + 1 + 2 + 2 +(4 rows) +1: select * from t_rename2 order by a; + a +--- + 1 + 2 +(2 rows) + +-- recovery the nodes +!\retcode gprecoverseg -a; +(exited with code 0) +1: select wait_until_segment_synchronized(0); + wait_until_segment_synchronized +--------------------------------- + OK +(1 row) + +!\retcode gprecoverseg -ar; +(exited with code 0) +1: select wait_until_segment_synchronized(0); + wait_until_segment_synchronized +--------------------------------- + OK +(1 row) + +-- verify the first segment is recovered to the original state. +1: select role, preferred_role from gp_segment_configuration where content = 0 order by role; + role | preferred_role +------+---------------- + m | m + p | p +(2 rows) + +-- cleanup +1: drop table t_rename1; +DROP +1: drop table t_rename2; +DROP +!\retcode gpconfig -r gp_gang_creation_retry_count --skipvalidation; +(exited with code 0) +!\retcode gpconfig -r gp_gang_creation_retry_timer --skipvalidation; +(exited with code 0) +!\retcode gpconfig -r archive_mode --skipvalidation; +(exited with code 0) +!\retcode gpconfig -r archive_command --skipvalidation; +(exited with code 0) +!\retcode gpstop -rai; +(exited with code 0) diff --git a/src/test/isolation2/isolation2_schedule b/src/test/isolation2/isolation2_schedule index 18fd4a2c0c4..f9e6082ff45 100644 --- a/src/test/isolation2/isolation2_schedule +++ b/src/test/isolation2/isolation2_schedule @@ -250,6 +250,7 @@ test: segwalrep/dtm_recovery_on_standby test: segwalrep/commit_blocking_on_standby test: segwalrep/dtx_recovery_wait_lsn test: segwalrep/select_throttle +test: segwalrep/startup_rename_prepared_xlog test: fts_manual_probe test: fts_session_reset # unstable FTS test in different arch diff --git a/src/test/isolation2/sql/segwalrep/startup_rename_prepared_xlog.sql b/src/test/isolation2/sql/segwalrep/startup_rename_prepared_xlog.sql new file mode 100644 index 00000000000..f612647f8ea --- /dev/null +++ b/src/test/isolation2/sql/segwalrep/startup_rename_prepared_xlog.sql @@ -0,0 +1,89 @@ +-- Test a bug where the last XLOG segment is renamed to .partial before prepared +-- transactions contained in it are recovered. This would result in segfault at +-- RecoverPreparedTransactions. The failure is well described in PG hackers: +-- https://www.postgresql.org/message-id/743b9b45a2d4013bd90b6a5cba8d6faeb717ee34.camel%40cybertec.at +-- However, in GP due to an older version of PG this error is masked by always caching +-- the last opened xlog segment in XLogRead(). To actually trigger the failure we need to +-- create unfinished prepared transactions in TWO different xlog segments. + +-- Allow extra time for mirror promotion to complete recovery to avoid +-- gprecoverseg BEGIN failures due to gang creation failure as some primaries +-- are not up. Setting these increase the number of retries in gang creation in +-- case segment is in recovery. Approximately we want to wait 30 seconds. +!\retcode gpconfig -c gp_gang_creation_retry_count -v 120 --skipvalidation --masteronly; +!\retcode gpconfig -c gp_gang_creation_retry_timer -v 1000 --skipvalidation --masteronly; + +-- The last xlog segment is renamed to .partial only when archive mode is on +!\retcode gpconfig -c archive_mode -v on; +!\retcode gpconfig -c archive_command -v '/bin/true'; +!\retcode gpstop -rai; + +1: create extension if not exists gp_inject_fault; +1: create table t_rename1 (a int); +1: create table t_rename2 (a int); + +-- generate an orphaned prepare transaction in first segment +1: select gp_inject_fault('dtm_broadcast_prepare', 'suspend', dbid) + from gp_segment_configuration where role = 'p' and content = -1; +-- assume (2), (1) are on different segments and one tuple is on the first segment. +-- the test finally double-check that. +2&: insert into t_rename1 values(2),(1); +1: select gp_wait_until_triggered_fault('dtm_broadcast_prepare', 1, dbid) + from gp_segment_configuration where role = 'p' and content = -1; + +-- start another xlog segment (will be renamed to .partial on failover) +-- start_ignore +0U: select pg_switch_xlog(); +-- end_ignore + +-- inject another fault and prepare transaction in the new xlog segment +1: select gp_inject_fault('dtm_broadcast_prepare', 'reset', dbid) + from gp_segment_configuration where role = 'p' and content = -1; +1: select gp_inject_fault('dtm_broadcast_prepare', 'suspend', dbid) + from gp_segment_configuration where role = 'p' and content = -1; +-- assume (4), (3) are on different segments and one tuple is on the first segment. +-- the test finally double-check that. +3&: insert into t_rename2 values(2),(1); +1: select gp_wait_until_triggered_fault('dtm_broadcast_prepare', 1, dbid) + from gp_segment_configuration where role = 'p' and content = -1; + +-- shutdown primary and make sure the segment is down +-1U: select pg_ctl((SELECT datadir from gp_segment_configuration c + where c.role='p' and c.content=0), 'stop', 'immediate'); +1: select gp_request_fts_probe_scan(); +1: select role, preferred_role from gp_segment_configuration where content = 0 order by role; + +-- double confirm that promote succeeds. +-- also double confirm that +-- 1. tuples (2) and (1) are located on two segments (thus we are testing 2pc with prepared transaction). +-- 2. there are tuples on the first segment (we have been testing on the first segment). +1: insert into t_rename1 values(2),(1); +1: select gp_segment_id, * from t_rename1 order by a; + +1: select gp_inject_fault('dtm_broadcast_prepare', 'reset', dbid) + from gp_segment_configuration where role = 'p' and content = -1; +2<: +3<: + +-- confirm the "orphaned" prepared trnasaction commits finally. +1: select * from t_rename1 order by a; +1: select * from t_rename2 order by a; + +-- recovery the nodes +!\retcode gprecoverseg -a; +1: select wait_until_segment_synchronized(0); + +!\retcode gprecoverseg -ar; +1: select wait_until_segment_synchronized(0); + +-- verify the first segment is recovered to the original state. +1: select role, preferred_role from gp_segment_configuration where content = 0 order by role; + +-- cleanup +1: drop table t_rename1; +1: drop table t_rename2; +!\retcode gpconfig -r gp_gang_creation_retry_count --skipvalidation; +!\retcode gpconfig -r gp_gang_creation_retry_timer --skipvalidation; +!\retcode gpconfig -r archive_mode --skipvalidation; +!\retcode gpconfig -r archive_command --skipvalidation; +!\retcode gpstop -rai; From fbe719200eaa7bef531e2ea599134b2001f14796 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Tue, 10 Jun 2025 11:32:11 +0800 Subject: [PATCH 009/244] [AQUMV] Remove mutable checks during answering query. The mutable checks are redundant at this stage since we already perform this validation during gp_matview_aux entity insertion. Authored-by: Zhang Mingli avamingli@gmail.com --- src/backend/catalog/gp_matview_aux.c | 9 ++++++++- src/backend/optimizer/plan/aqumv.c | 13 ------------- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/src/backend/catalog/gp_matview_aux.c b/src/backend/catalog/gp_matview_aux.c index e62486f2488..9e1c224019c 100644 --- a/src/backend/catalog/gp_matview_aux.c +++ b/src/backend/catalog/gp_matview_aux.c @@ -98,7 +98,14 @@ GetViewBaseRelids(const Query *viewQuery, bool *has_foreign) return NIL; } - /* As we will use views, make it strict to unmutable. */ + /* + * Use immutable functions for views to ensure strict immutability. + * While STABLE functions don't modify the database and return consistent + * results for the same arguments within a statement, AQUMV rewrites + * the query to new SQL. The behavior with STABLE functions isn't entirely + * clear in this context, so we're conservatively requiring IMMUTABLE + * functions for now. + */ if (contain_mutable_functions((Node*)viewQuery)) return NIL; diff --git a/src/backend/optimizer/plan/aqumv.c b/src/backend/optimizer/plan/aqumv.c index 59d133d8af4..63e82d759b0 100644 --- a/src/backend/optimizer/plan/aqumv.c +++ b/src/backend/optimizer/plan/aqumv.c @@ -353,19 +353,6 @@ answer_query_using_materialized_views(PlannerInfo *root, AqumvContext aqumv_cont subroot->tuple_fraction = root->tuple_fraction; subroot->limit_tuples = root->limit_tuples; - /* - * AQUMV_FIXME_MVP - * Are stable functions OK? - * A STABLE function cannot modify the database and is guaranteed to - * return the same results given the same arguments for all rows - * within a single statement. - * But AQUMV rewrites the query to a new SQL actually, though the same - * results is guaranteed. - * Its's unclear whether STABLE is OK, let's be conservative for now. - */ - if(contain_mutable_functions((Node *)viewQuery)) - continue; - context = aqumv_init_context(viewQuery->targetList, matviewRel->rd_att); if (!parse->hasAggs && viewQuery->hasAggs) From 3eb74b801af88681707f84dd73179a1f187fa952 Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Wed, 11 Jun 2025 15:47:19 +0800 Subject: [PATCH 010/244] Fix unordered GUC names. As the comments said: items in this file should be ordered. Authored-by: Zhang Mingli avamingli@gmail.com --- src/include/utils/sync_guc_name.h | 8 ++++---- src/include/utils/unsync_guc_name.h | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/include/utils/sync_guc_name.h b/src/include/utils/sync_guc_name.h index 286762e6d6f..7cecf083727 100644 --- a/src/include/utils/sync_guc_name.h +++ b/src/include/utils/sync_guc_name.h @@ -53,6 +53,8 @@ "gp_appendonly_compaction", "gp_appendonly_compaction_segfile_limit", "gp_appendonly_compaction_threshold", + "gp_appendonly_insert_files", + "gp_appendonly_insert_files_tuples_range", "gp_appendonly_verify_block_checksums", "gp_appendonly_verify_write_block", "gp_blockdirectory_entry_min_range", @@ -63,6 +65,7 @@ "gp_disable_tuple_hints", "gp_enable_interconnect_aggressive_retry", "gp_enable_runtime_filter", + "gp_enable_runtime_filter_pushdown", "gp_enable_segment_copy_checking", "gp_external_enable_filter_pushdown", "gp_hashjoin_tuples_per_bucket", @@ -95,6 +98,7 @@ "gp_log_suboverflow_statement", "gp_max_packet_size", "gp_motion_slice_noop", + "gp_random_insert_segments", "gp_resgroup_debug_wait_queue", "gp_resgroup_memory_policy_auto_fixed_mem", "gp_resqueue_memory_policy_auto_fixed_mem", @@ -193,8 +197,4 @@ "wal_debug", "wal_sender_timeout", "work_mem", - "gp_appendonly_insert_files", - "gp_appendonly_insert_files_tuples_range", - "gp_enable_runtime_filter_pushdown", - "gp_random_insert_segments", "zero_damaged_pages", diff --git a/src/include/utils/unsync_guc_name.h b/src/include/utils/unsync_guc_name.h index e7be8afc7be..959d2a89a73 100644 --- a/src/include/utils/unsync_guc_name.h +++ b/src/include/utils/unsync_guc_name.h @@ -7,7 +7,6 @@ *-------------------------------------------------------------------------- */ /* items in this file should be ordered */ - "enable_answer_query_using_materialized_views", "application_name", "aqumv_allow_foreign_table", "archive_cleanup_command", @@ -44,6 +43,7 @@ "checkpoint_warning", "client_connection_check_interval", "client_encoding", + "cluster_key_command", "cluster_name", "compute_query_id", "config_file", @@ -106,6 +106,7 @@ "dynamic_shared_memory_type", "effective_cache_size", "effective_io_concurrency", + "enable_answer_query_using_materialized_views", "enable_async_append", "enable_bitmapscan", "enable_gathermerge", @@ -140,6 +141,7 @@ "exit_on_error", "external_pid_file", "extra_float_digits", + "file_encryption_method", "from_collapse_limit", "fsync", "full_page_writes", @@ -572,6 +574,7 @@ "task_log_statement", "task_use_background_worker", "task_timezone", + "tde_force_switch", "temp_file_limit", "test_AppendOnlyHash_eviction_vs_just_marking_not_inuse", "test_print_direct_dispatch_info", @@ -623,6 +626,3 @@ "xid_warn_limit", "xmlbinary", "xmloption", - "cluster_key_command", - "file_encryption_method", - "tde_force_switch", From ff9375e73181648d67cf98fd0a0488fad15c22cf Mon Sep 17 00:00:00 2001 From: Dianjin Wang Date: Mon, 16 Jun 2025 14:07:52 +0800 Subject: [PATCH 011/244] Update bash $PATH in gpdemo for better portability Replace hardcoded `/usr/bin/bash` path with `/usr/bin/env bash` in gpAux/gpdemo/Makefile to improve portability across different systems. This change ensures the demo cluster can be created regardless of where bash is installed on the system. Will return the following errors when testing under Ubuntu 18.04 container: ``` gpadmin@cdw:~/cloudberry$ make create-demo-cluster -C ~/cloudberry make: Entering directory '/home/gpadmin/cloudberry' make -C gpAux/gpdemo create-demo-cluster make[1]: Entering directory '/home/gpadmin/cloudberry/gpAux/gpdemo' make[1]: /usr/bin/bash: Command not found ``` This change makes the demo environment more robust across different Linux distributions, container environments, and custom installations. --- gpAux/gpdemo/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gpAux/gpdemo/Makefile b/gpAux/gpdemo/Makefile index b6e794e57ab..386876b9b87 100644 --- a/gpAux/gpdemo/Makefile +++ b/gpAux/gpdemo/Makefile @@ -9,7 +9,7 @@ top_builddir = ../.. -include $(top_builddir)/src/Makefile.global -SHELL := /usr/bin/bash +SHELL := /usr/bin/env bash all: $(MAKE) clean From 851373b6849380b377841d08e82bf3e90a1bf4ca Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Mon, 16 Jun 2025 10:40:01 +0800 Subject: [PATCH 012/244] Fix gcc 12.4 complier error in LookupFuncWithArgs. --- src/backend/parser/parse_func.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/backend/parser/parse_func.c b/src/backend/parser/parse_func.c index bbda4328d4c..44acfd9c81b 100644 --- a/src/backend/parser/parse_func.c +++ b/src/backend/parser/parse_func.c @@ -2260,7 +2260,7 @@ LookupFuncName(List *funcname, int nargs, const Oid *argtypes, bool missing_ok) Oid LookupFuncWithArgs(ObjectType objtype, ObjectWithArgs *func, bool missing_ok) { - Oid argoids[FUNC_MAX_ARGS]; + Oid argoids[FUNC_MAX_ARGS] = {InvalidOid,}; int argcount; int nargs; int i; From a8e1ab66a7ce4d796b8eea49b1100599a5ff2b5d Mon Sep 17 00:00:00 2001 From: Zhang Mingli Date: Thu, 5 Jun 2025 14:19:35 +0800 Subject: [PATCH 013/244] [AQUMV] Directly compute queries from materialized views with GROUP BY. This commit enhances the AQUMV system by enabling it to compute queries directly from materialized views that already contain a GROUP BY clause. This improvement allows us to bypass additional GROUP BY operations during query execution, resulting in faster and more efficient performance. For example, with a materialized view defined as follows: ```sql CREATE MATERIALIZED VIEW mv_group_1 AS SELECT c, b, COUNT(b) AS count_b FROM t0 WHERE a > 3 GROUP BY c, b; ``` An original query like: ```sql SELECT COUNT(b), b, c FROM t0 WHERE a > 3 GROUP BY b, c; ``` is rewritten to: ```sql SELECT count_b, b, c FROM mv_group_1; ``` The plan looks like: ```sql explain(costs off, verbose) select count(b), b, c from t0 where a > 3 group by b, c; QUERY PLAN --------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) Output: count, b, c -> Seq Scan on aqumv.mv_group_1 Output: count, b, c Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' Optimizer: Postgres query optimizer (6 rows) ``` The two SQL queries yield equivalent results, even though the selected columns are in a different order. Since mv_group_1 already contains the aggregated results and all rows have a column a value greater than 3, there is no need for additional filtering or GROUP BY operations. This enhancement eliminates redundant computations, leading to significant time savings. Fetching results directly from these views reduces overall execution time, improving responsiveness for complex queries. This is particularly beneficial for large datasets, allowing efficient data analysis without performance degradation. The feature also applies to Dynamic Tables and Incremental Materialized Views. Authored-by: Zhang Mingli avamingli@gmail.com --- src/backend/optimizer/plan/aqumv.c | 110 ++++++- src/test/regress/expected/aqumv.out | 471 ++++++++++++++++++++++++++++ src/test/regress/sql/aqumv.sql | 100 ++++++ 3 files changed, 676 insertions(+), 5 deletions(-) diff --git a/src/backend/optimizer/plan/aqumv.c b/src/backend/optimizer/plan/aqumv.c index 63e82d759b0..4a576061780 100644 --- a/src/backend/optimizer/plan/aqumv.c +++ b/src/backend/optimizer/plan/aqumv.c @@ -74,6 +74,13 @@ static Node *aqumv_adjust_sub_matched_expr_mutator(Node *node, aqumv_equivalent_ static bool contain_var_or_aggstar_clause_walker(Node *node, void *context); static bool check_partition(Query *parse, Oid origin_rel_oid); +static bool +groupby_query_rewrite(PlannerInfo *subroot, + Query *parse, + Query *viewQuery, + aqumv_equivalent_transformation_context *context, + AqumvContext aqumv_context); + typedef struct { int complexity; @@ -358,19 +365,23 @@ answer_query_using_materialized_views(PlannerInfo *root, AqumvContext aqumv_cont if (!parse->hasAggs && viewQuery->hasAggs) continue; - if (parse->hasAggs && viewQuery->hasAggs) + if (parse->groupClause != NIL && viewQuery->groupClause != NIL) + { + if (!groupby_query_rewrite(subroot, parse, viewQuery, context, aqumv_context)) + continue; + } + else if (parse->hasAggs && viewQuery->hasAggs) { + /* Both don't have group by. */ + { if (parse->hasDistinctOn || parse->distinctClause != NIL || - parse->groupClause != NIL || /* TODO: GROUP BY */ parse->groupingSets != NIL || parse->groupDistinct) continue; - /* No Group by now. */ if (viewQuery->hasDistinctOn || viewQuery->distinctClause != NIL || - viewQuery->groupClause != NIL || viewQuery->groupingSets != NIL || viewQuery->groupDistinct || viewQuery->havingQual != NULL || /* HAVING clause is not supported on IMMV yet. */ @@ -388,7 +399,7 @@ answer_query_using_materialized_views(PlannerInfo *root, AqumvContext aqumv_cont */ if (parse->sortClause != NIL || viewQuery->sortClause != NIL) { - /* Earse view's sort caluse, it's ok to let alone view's target list. */ + /* Erase view's sort caluse, it's ok to let alone view's target list. */ viewQuery->sortClause = NIL; } @@ -463,6 +474,7 @@ answer_query_using_materialized_views(PlannerInfo *root, AqumvContext aqumv_cont /* Select from a mv never have that.*/ subroot->append_rel_list = NIL; + } } else { @@ -896,3 +908,91 @@ check_partition(Query *parse, Oid origin_rel_oid) } return true; } + +static bool +groupby_query_rewrite(PlannerInfo *subroot, + Query *parse, + Query *viewQuery, + aqumv_equivalent_transformation_context *context, + AqumvContext aqumv_context) +{ + List *post_quals = NIL; + List *mv_final_tlist = NIL; + + if (!parse->hasAggs || !viewQuery->hasAggs) + return false; + + /* Both have Group by and aggregation. */ + if (parse->groupClause == NIL || viewQuery->groupClause == NIL) + return false; + + if (parse->hasDistinctOn || + parse->distinctClause != NIL || + parse->groupingSets != NIL || + parse->sortClause != NIL || + limit_needed(parse) || + parse->havingQual != NULL || + parse->groupDistinct) + return false; + + if (viewQuery->hasDistinctOn || + viewQuery->distinctClause != NIL || + viewQuery->groupingSets != NIL || + viewQuery->groupDistinct || + viewQuery->havingQual != NULL || + viewQuery->sortClause != NIL || + limit_needed(viewQuery)) + return false; + + if (tlist_has_srf(parse)) + return false; + + preprocess_qual_conditions(subroot, (Node *) viewQuery->jointree); + + if(!aqumv_process_from_quals(parse->jointree->quals, viewQuery->jointree->quals, &post_quals)) + return false; + + if (post_quals != NIL) + return false; + + /* + * There should be no post_quals for now, erase those from view. + */ + viewQuery->jointree->quals = NULL; + + if (list_difference(parse->groupClause, viewQuery->groupClause)) + return false; + + if (list_difference(viewQuery->groupClause, parse->groupClause)) + return false; + + /* + * Group By clauses are equal, erase those from view. + */ + viewQuery->groupClause = NIL; + + if(!aqumv_process_targetlist(context, aqumv_context->raw_processed_tlist, &mv_final_tlist)) + return false; + + viewQuery->targetList = mv_final_tlist; + /* SRF is not supported now, but correct the field. */ + viewQuery->hasTargetSRFs = parse->hasTargetSRFs; + viewQuery->hasAggs = false; + subroot->agginfos = NIL; + subroot->aggtransinfos = NIL; + subroot->hasNonPartialAggs = false; + subroot->hasNonSerialAggs = false; + subroot->numOrderedAggs = false; + /* CBDB specifical */ + subroot->hasNonCombine = false; + subroot->numPureOrderedAggs = false; + /* + * NB: Update processed_tlist again in case that tlist has been changed. + */ + subroot->processed_tlist = NIL; + preprocess_targetlist(subroot); + + /* Select from a mv never have that.*/ + subroot->append_rel_list = NIL; + return true; +} diff --git a/src/test/regress/expected/aqumv.out b/src/test/regress/expected/aqumv.out index 463b0b37032..926ef05cd32 100644 --- a/src/test/regress/expected/aqumv.out +++ b/src/test/regress/expected/aqumv.out @@ -3374,6 +3374,477 @@ select * from t_insert; 1000 (1 row) +abort; +-- Test view has Group By +begin; +create table t0 as select i as a, i+1 as b , i+2 as c, i+3 as d from generate_series(1, 5) i; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'a' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +analyze t0; +create materialized view mv_group_0 as select c, b, sum(a), count(b) from t0 group by b, c; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'b, c' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +create materialized view mv_group_1 as select c, b, count(b) from t0 where a > 3 group by c, b; +NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column(s) named 'c, b' as the Apache Cloudberry data distribution key for this table. +HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. +analyze mv_group_0; +analyze mv_group_1; +-- no qual, exactly match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select c, b, sum(a), count(b) from t0 group by b, c; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: c, b, (sum(a)), (count(b)) + -> Finalize HashAggregate + Output: c, b, sum(a), count(b) + Group Key: t0.b, t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: c, b, (PARTIAL sum(a)), (PARTIAL count(b)) + Hash Key: b, c + -> Partial HashAggregate + Output: c, b, PARTIAL sum(a), PARTIAL count(b) + Group Key: t0.b, t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(15 rows) + +select c, b, sum(a), count(b) from t0 group by b, c; + c | b | sum | count +---+---+------+------- + 6 | 5 | 2048 | 512 + 3 | 2 | 512 | 512 + 5 | 4 | 1536 | 512 + 4 | 3 | 1024 | 512 + 7 | 6 | 2560 | 512 +(5 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select c, b, sum(a), count(b) from t0 group by b, c; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: c, b, sum, count + -> Seq Scan on aqumv.mv_group_0 + Output: c, b, sum, count + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(6 rows) + +select c, b, sum(a), count(b) from t0 group by b, c; + c | b | sum | count +---+---+------+------- + 6 | 5 | 2048 | 512 + 3 | 2 | 512 | 512 + 5 | 4 | 1536 | 512 + 4 | 3 | 1024 | 512 + 7 | 6 | 2560 | 512 +(5 rows) + +-- no qual, different order +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select b, sum(a), c, count(b) from t0 group by c, b; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: b, (sum(a)), c, (count(b)) + -> Finalize HashAggregate + Output: b, sum(a), c, count(b) + Group Key: t0.c, t0.b + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: b, c, (PARTIAL sum(a)), (PARTIAL count(b)) + Hash Key: c, b + -> Partial HashAggregate + Output: b, c, PARTIAL sum(a), PARTIAL count(b) + Group Key: t0.c, t0.b + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(15 rows) + +select b, sum(a), c, count(b) from t0 group by c, b; + b | sum | c | count +---+------+---+------- + 6 | 2560 | 7 | 512 + 4 | 1536 | 5 | 512 + 5 | 2048 | 6 | 512 + 3 | 1024 | 4 | 512 + 2 | 512 | 3 | 512 +(5 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select b, sum(a), c, count(b) from t0 group by c, b; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: b, sum, c, count + -> Seq Scan on aqumv.mv_group_0 + Output: b, sum, c, count + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(6 rows) + +select b, sum(a), c, count(b) from t0 group by c, b; + b | sum | c | count +---+------+---+------- + 5 | 2048 | 6 | 512 + 2 | 512 | 3 | 512 + 4 | 1536 | 5 | 512 + 3 | 1024 | 4 | 512 + 6 | 2560 | 7 | 512 +(5 rows) + +-- no qual, different expr +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select b + c + 1, sum(a) + count(b) from t0 group by c, b; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: (((b + c) + 1)), ((sum(a) + count(b))), c, b + -> Finalize HashAggregate + Output: ((b + c) + 1), (sum(a) + count(b)), c, b + Group Key: t0.c, t0.b + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: c, b, (PARTIAL sum(a)), (PARTIAL count(b)) + Hash Key: c, b + -> Partial HashAggregate + Output: c, b, PARTIAL sum(a), PARTIAL count(b) + Group Key: t0.c, t0.b + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(15 rows) + +select b + c + 1, sum(a) + count(b) from t0 group by c, b; + ?column? | ?column? +----------+---------- + 8 | 1536 + 14 | 3072 + 10 | 2048 + 12 | 2560 + 6 | 1024 +(5 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select b + c + 1, sum(a) + count(b) from t0 group by c, b; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: (((b + c) + 1)), ((sum + count)), c, b + -> Seq Scan on aqumv.mv_group_0 + Output: ((b + c) + 1), (sum + count), c, b + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(6 rows) + +select b + c + 1, sum(a) + count(b) from t0 group by c, b; + ?column? | ?column? +----------+---------- + 8 | 1536 + 14 | 3072 + 12 | 2560 + 6 | 1024 + 10 | 2048 +(5 rows) + +-- no qual, should not match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select c, count(b) from t0 group by c ; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: c, (count(b)) + -> Finalize HashAggregate + Output: c, count(b) + Group Key: t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: c, (PARTIAL count(b)) + Hash Key: c + -> Partial HashAggregate + Output: c, PARTIAL count(b) + Group Key: t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(15 rows) + +select c, count(b) from t0 group by c ; + c | count +---+------- + 4 | 512 + 3 | 512 + 7 | 512 + 5 | 512 + 6 | 512 +(5 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select c, count(b) from t0 group by c ; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: c, (count(b)) + -> Finalize HashAggregate + Output: c, count(b) + Group Key: t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: c, (PARTIAL count(b)) + Hash Key: c + -> Partial HashAggregate + Output: c, PARTIAL count(b) + Group Key: t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(15 rows) + +select c, count(b) from t0 group by c ; + c | count +---+------- + 4 | 512 + 3 | 512 + 7 | 512 + 5 | 512 + 6 | 512 +(5 rows) + +-- with qual, exactly match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select c, b, count(b) from t0 where a > 3 group by c, b; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: c, b, (count(b)) + -> Finalize HashAggregate + Output: c, b, count(b) + Group Key: t0.c, t0.b + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: c, b, (PARTIAL count(b)) + Hash Key: c, b + -> Partial HashAggregate + Output: c, b, PARTIAL count(b) + Group Key: t0.c, t0.b + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Filter: (t0.a > 3) + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +select c, b, count(b) from t0 where a > 3 group by c, b; + c | b | count +---+---+------- + 7 | 6 | 512 + 6 | 5 | 512 +(2 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select c, b, count(b) from t0 where a > 3 group by c, b; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: c, b, count + -> Seq Scan on aqumv.mv_group_1 + Output: c, b, count + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(6 rows) + +select c, b, count(b) from t0 where a > 3 group by c, b; + c | b | count +---+---+------- + 7 | 6 | 512 + 6 | 5 | 512 +(2 rows) + +-- with qual, different order +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select count(b), b, c from t0 where a > 3 group by b, c; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: (count(b)), b, c + -> Finalize HashAggregate + Output: count(b), b, c + Group Key: t0.b, t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: b, c, (PARTIAL count(b)) + Hash Key: b, c + -> Partial HashAggregate + Output: b, c, PARTIAL count(b) + Group Key: t0.b, t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Filter: (t0.a > 3) + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +select count(b), b, c from t0 where a > 3 group by b, c; + count | b | c +-------+---+--- + 512 | 5 | 6 + 512 | 6 | 7 +(2 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select count(b), b, c from t0 where a > 3 group by b, c; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: count, b, c + -> Seq Scan on aqumv.mv_group_1 + Output: count, b, c + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(6 rows) + +select count(b), b, c from t0 where a > 3 group by b, c; + count | b | c +-------+---+--- + 512 | 6 | 7 + 512 | 5 | 6 +(2 rows) + +-- with qual, different expr +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: ((count(b) + 1)), ((b + 1)), c, b + -> Finalize HashAggregate + Output: (count(b) + 1), (b + 1), c, b + Group Key: t0.b, t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: c, b, (PARTIAL count(b)) + Hash Key: b, c + -> Partial HashAggregate + Output: c, b, PARTIAL count(b) + Group Key: t0.b, t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Filter: (t0.a > 3) + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; + ?column? | ?column? | c +----------+----------+--- + 513 | 6 | 6 + 513 | 7 | 7 +(2 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: ((count + 1)), ((b + 1)), c, b + -> Seq Scan on aqumv.mv_group_1 + Output: (count + 1), (b + 1), c, b + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(6 rows) + +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; + ?column? | ?column? | c +----------+----------+--- + 513 | 7 | 7 + 513 | 6 | 6 +(2 rows) + +-- with qual, should not match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: b, c, (count(b)) + -> Finalize HashAggregate + Output: b, c, count(b) + Group Key: t0.b, t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: b, c, (PARTIAL count(b)) + Hash Key: b, c + -> Partial HashAggregate + Output: b, c, PARTIAL count(b) + Group Key: t0.b, t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Filter: ((t0.a > 3) AND (t0.b > 1)) + Settings: enable_answer_query_using_materialized_views = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; + b | c | count +---+---+------- + 5 | 6 | 512 + 6 | 7 | 512 +(2 rows) + +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) + Output: b, c, (count(b)) + -> Finalize HashAggregate + Output: b, c, count(b) + Group Key: t0.b, t0.c + -> Redistribute Motion 3:3 (slice2; segments: 3) + Output: b, c, (PARTIAL count(b)) + Hash Key: b, c + -> Partial HashAggregate + Output: b, c, PARTIAL count(b) + Group Key: t0.b, t0.c + -> Seq Scan on aqumv.t0 + Output: a, b, c, d + Filter: ((t0.a > 3) AND (t0.b > 1)) + Settings: enable_answer_query_using_materialized_views = 'on', optimizer = 'off' + Optimizer: Postgres query optimizer +(16 rows) + +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; + b | c | count +---+---+------- + 5 | 6 | 512 + 6 | 7 | 512 +(2 rows) + abort; reset optimizer; reset enable_answer_query_using_materialized_views; diff --git a/src/test/regress/sql/aqumv.sql b/src/test/regress/sql/aqumv.sql index 87fe401c2ab..35690d220e8 100644 --- a/src/test/regress/sql/aqumv.sql +++ b/src/test/regress/sql/aqumv.sql @@ -874,6 +874,106 @@ insert into t_insert select count(a) from t_select; select * from t_insert; abort; +-- Test view has Group By +begin; +create table t0 as select i as a, i+1 as b , i+2 as c, i+3 as d from generate_series(1, 5) i; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +insert into t0 select * from t0; +analyze t0; +create materialized view mv_group_0 as select c, b, sum(a), count(b) from t0 group by b, c; +create materialized view mv_group_1 as select c, b, count(b) from t0 where a > 3 group by c, b; +analyze mv_group_0; +analyze mv_group_1; + +-- no qual, exactly match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select c, b, sum(a), count(b) from t0 group by b, c; +select c, b, sum(a), count(b) from t0 group by b, c; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select c, b, sum(a), count(b) from t0 group by b, c; +select c, b, sum(a), count(b) from t0 group by b, c; + +-- no qual, different order +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select b, sum(a), c, count(b) from t0 group by c, b; +select b, sum(a), c, count(b) from t0 group by c, b; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select b, sum(a), c, count(b) from t0 group by c, b; +select b, sum(a), c, count(b) from t0 group by c, b; + +-- no qual, different expr +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select b + c + 1, sum(a) + count(b) from t0 group by c, b; +select b + c + 1, sum(a) + count(b) from t0 group by c, b; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select b + c + 1, sum(a) + count(b) from t0 group by c, b; +select b + c + 1, sum(a) + count(b) from t0 group by c, b; + +-- no qual, should not match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select c, count(b) from t0 group by c ; +select c, count(b) from t0 group by c ; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select c, count(b) from t0 group by c ; +select c, count(b) from t0 group by c ; + +-- with qual, exactly match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select c, b, count(b) from t0 where a > 3 group by c, b; +select c, b, count(b) from t0 where a > 3 group by c, b; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select c, b, count(b) from t0 where a > 3 group by c, b; +select c, b, count(b) from t0 where a > 3 group by c, b; + +-- with qual, different order +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select count(b), b, c from t0 where a > 3 group by b, c; +select count(b), b, c from t0 where a > 3 group by b, c; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select count(b), b, c from t0 where a > 3 group by b, c; +select count(b), b, c from t0 where a > 3 group by b, c; + +-- with qual, different expr +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; +select count(b) + 1, b + 1, c from t0 where a > 3 group by b, c; + +-- with qual, should not match +set local enable_answer_query_using_materialized_views = off; +explain(costs off, verbose) +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; +set local enable_answer_query_using_materialized_views = on; +explain(costs off, verbose) +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; +select b, c, count(b) from t0 where a > 3 and b > 1 group by b, c; + +abort; + reset optimizer; reset enable_answer_query_using_materialized_views; -- start_ignore From 1302354437ec281a3841b8cd8e630a3dcce590fa Mon Sep 17 00:00:00 2001 From: zhoujiaqi Date: Tue, 27 May 2025 18:34:29 +0800 Subject: [PATCH 014/244] ORCA: deduplicate superset in AndOp when scalar condition has subset This commit addresses scenarios where certain WHERE conditions are subsets of others. For example, in WHERE A AND B, if condition A is a superset of B, we can direct prune the condition A. For most queries, eliminating redundant WHERE conditions to avoids unnecessary inner joins when subqueries exist in the WHERE clause. Due to the complexity of SQL, we can only target some simple cases as superset. And only when subqueries exist, deduplication can achieve actual performance improvement. In this commit, the selection for superset is: subqueries without `GROUP BY, WHERE, ORDER BY, LIMIT` as r-value. And the selection for subset is: WHERE conditions containing the same l-value as superset (or superset's l-value as joinkey with swapped values) and the subquery with `GROUP BY, WHERE, ORDER BY, LIMIT` as l-values. Of course, if the same supersets exist, it will also be pruned. Example: ``` the schema: t1(v1, v2), t2(v3, v4) The query: select * from t1,t2 where t1.v1 in (select v3 from t2) and t1.v1 in (select v3 from t2 where v3 < 100); equivalent to: select * from t1,t2 where t1.v1 t1.v1 in (select v3 from t2 where v3 < 100); ``` --- .../libgpopt/include/gpopt/base/CColRefSet.h | 4 + .../operators/CDedupSupersetPreprocessor.h | 126 ++ .../gporca/libgpopt/src/base/CColRefSet.cpp | 27 + .../operators/CDedupSupersetPreprocessor.cpp | 551 +++++ .../src/operators/CExpressionPreprocessor.cpp | 13 +- .../gporca/libgpopt/src/operators/Makefile | 3 +- src/test/regress/expected/cte_prune.out | 6 +- .../regress/expected/cte_prune_optimizer.out | 6 +- src/test/regress/expected/dedupset.out | 1936 +++++++++++++++++ .../regress/expected/dedupset_optimizer.out | 1614 ++++++++++++++ src/test/regress/greenplum_schedule | 2 + src/test/regress/sql/cte_prune.sql | 6 +- src/test/regress/sql/dedupset.sql | 174 ++ 13 files changed, 4456 insertions(+), 12 deletions(-) create mode 100644 src/backend/gporca/libgpopt/include/gpopt/operators/CDedupSupersetPreprocessor.h create mode 100644 src/backend/gporca/libgpopt/src/operators/CDedupSupersetPreprocessor.cpp create mode 100644 src/test/regress/expected/dedupset.out create mode 100644 src/test/regress/expected/dedupset_optimizer.out create mode 100644 src/test/regress/sql/dedupset.sql diff --git a/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h b/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h index b4e34c62c81..1af8748a5c3 100644 --- a/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h +++ b/src/backend/gporca/libgpopt/include/gpopt/base/CColRefSet.h @@ -133,6 +133,10 @@ class CColRefSet : public CBitSet // extract all column ids void ExtractColIds(CMemoryPool *mp, ULongPtrArray *colids) const; + // extract the index(relative position) in current CColRefSet + // notice that: Whenever the current CColRefSet changes, the index will also change. + void ExtractIndex(const CColRef *querycr, ULONG *colidx) const; + // are the columns in the column reference set covered by the array of column ref sets static BOOL FCovered(CColRefSetArray *pdrgpcrs, CColRefSet *pcrs); diff --git a/src/backend/gporca/libgpopt/include/gpopt/operators/CDedupSupersetPreprocessor.h b/src/backend/gporca/libgpopt/include/gpopt/operators/CDedupSupersetPreprocessor.h new file mode 100644 index 00000000000..d9255212c73 --- /dev/null +++ b/src/backend/gporca/libgpopt/include/gpopt/operators/CDedupSupersetPreprocessor.h @@ -0,0 +1,126 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * CDedupSupersetPreprocessor.h + * + * IDENTIFICATION + * src/backend/gporca/libgpopt/include/gpopt/operators/CDedupSupersetPreprocessor.h + * + *------------------------------------------------------------------------- + */ +#ifndef GPOPT_CDedupSupersetPreprocessor_H +#define GPOPT_CDedupSupersetPreprocessor_H + +#include "gpos/base.h" +#include "gpos/memory/set.h" + +#include "gpopt/base/CColRefSet.h" +#include "gpopt/operators/CExpression.h" + +namespace gpopt +{ + +class TableDescIdent final +{ +public: + IMDId *pmdid; + ULONG subid; + ULONG ocolid; + + TableDescIdent(IMDId *mdid, ULONG sid, ULONG output_colid); + + ~TableDescIdent(); + + // hash function + static ULONG HashFunc(const TableDescIdent *ptdi); + + // equality function + static BOOL EqualFunc(const TableDescIdent *pltdi, + const TableDescIdent *prtdi); +}; + +// desc to ulong map +using TDIToUlongPtrMap = + CHashMap, + CleanupDelete>; + +// iterator +using TDIToUlongPtrMapIter = + CHashMapIter, + CleanupDelete>; + +class CDedupSupersetPreprocessor +{ +private: + // Expression is CLogicalGet,CLogicalDynamicGet or CLogicalCTEConsumer? + static BOOL PexprIsLogicalOutput(CExpression *pexpr); + + // Get the meta id from expression + static IMDId *PexprGetIMDid(CMemoryPool *mp, CExpression *pexpr); + + // full the maps with superset expression + static void ChildExprFullSuperset(CMemoryPool *mp, CExpression *pexpr, + TDIToUlongPtrMap *eqcrmaps, + BOOL *dedupulmasks, ULONG ul); + + // full the dedups array with subset(simply logicalget) + static void ChildExprFullSimplySubset(CMemoryPool *mp, CExpression *pexpr, + TDIToUlongPtrMap *eqcrmaps, + BOOL *dedupulmasks); + + // Extract the sub child from the join(inner join) expr + static CExpression *PexprExtractSubChildJoinExpr(CExpression *pexpr, + const CColRef *subany_col); + + // query the join subset + static ULONG *DriveFullJoinSubset(CMemoryPool *mp, + TDIToUlongPtrMap *eqcrmaps, + CExpression *pexpr_join, + const CColRef *pcr_sbany, + const CColRef *pcr_ident); + + // full the dedups array with subset(inner join) + static void ChildExprFullJoinSubset(CMemoryPool *mp, CExpression *pexpr, + TDIToUlongPtrMap *eqcrmaps, + BOOL *dedupulmasks); + + // child expr in CScalarBoolOp(ANDOP) is the superset + static BOOL ChildExprIsSuperset(CExpression *pexpr); + + // child expr in CScalarBoolOp(ANDOP) is the simply subset + static BOOL ChildExprIsSimplySubset(CExpression *pexpr); + + // child expr in CScalarBoolOp(ANDOP) is the inner-join subset + // left-semi-join won't occur in the pre-process + static BOOL ChildExprIsJoinSubset(CExpression *pexpr); + + +public: + CDedupSupersetPreprocessor(const CDedupSupersetPreprocessor &) = delete; + + // main driver + static CExpression *PexprPreprocess(CMemoryPool *mp, CExpression *pexpr); +}; + +} // namespace gpopt + + +#endif // GPOPT_CDedupSupersetPreprocessor_H diff --git a/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp b/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp index 09459441e5c..0555a7f2efd 100644 --- a/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp +++ b/src/backend/gporca/libgpopt/src/base/CColRefSet.cpp @@ -403,6 +403,33 @@ CColRefSet::ExtractColIds(CMemoryPool *mp, ULongPtrArray *colids) const } } +//--------------------------------------------------------------------------- +// @function: +// CColRefSet::ExtractColIds +// +// @doc: +// extract the index(relative position) in current CColRefSet +// notice that: Whenever the current CColRefSet changes, the index will also change. +// +//--------------------------------------------------------------------------- +void +CColRefSet::ExtractIndex(const CColRef *querycr, ULONG *colidx) const +{ + GPOS_ASSERT(nullptr != querycr); + GPOS_ASSERT(FMember(querycr)); + *colidx = 0; + CColRefSetIter crsi(*this); + while (crsi.Advance()) + { + CColRef *colref = crsi.Pcr(); + if (colref == querycr) { + return; + } + *colidx += 1; + } + *colidx = gpos::ulong_max; +} + //--------------------------------------------------------------------------- // @function: // CColRefSet::FContained diff --git a/src/backend/gporca/libgpopt/src/operators/CDedupSupersetPreprocessor.cpp b/src/backend/gporca/libgpopt/src/operators/CDedupSupersetPreprocessor.cpp new file mode 100644 index 00000000000..a0678af79ab --- /dev/null +++ b/src/backend/gporca/libgpopt/src/operators/CDedupSupersetPreprocessor.cpp @@ -0,0 +1,551 @@ +/*------------------------------------------------------------------------- + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + * CDedupSupersetPreprocessor.cpp + * + * IDENTIFICATION + * src/backend/gporca/libgpopt/src/operators/CDedupSupersetPreprocessor.cpp + * + *------------------------------------------------------------------------- + */ +#include "gpopt/operators/CDedupSupersetPreprocessor.h" + +#include + +#include "gpopt/operators/CExpression.h" +#include "gpopt/operators/CLogicalCTEAnchor.h" +#include "gpopt/operators/CLogicalCTEConsumer.h" +#include "gpopt/operators/CLogicalCTEProducer.h" +#include "gpopt/operators/CLogicalDynamicGet.h" +#include "gpopt/operators/CLogicalGet.h" +#include "gpopt/operators/CLogicalInnerJoin.h" +#include "gpopt/operators/CLogicalNAryJoin.h" +#include "gpopt/operators/CScalarCmp.h" +#include "gpopt/operators/CScalarIdent.h" +#include "gpopt/operators/CScalarSubqueryAny.h" +#include "gpopt/optimizer/COptimizerConfig.h" + +using namespace gpopt; + +#define CTEID_TO_OID_START 60000 + +TableDescIdent::TableDescIdent(IMDId *mdid, ULONG sid, ULONG output_colid) + : pmdid(mdid), subid(sid), ocolid(output_colid) +{ + GPOS_ASSERT(nullptr != mdid); +} + +TableDescIdent::~TableDescIdent() +{ + pmdid->Release(); +} + +ULONG +TableDescIdent::HashFunc(const TableDescIdent *ptdi) +{ + GPOS_ASSERT(nullptr != ptdi); + + return gpos::CombineHashes( + ptdi->pmdid->HashValue(), + gpos::CombineHashes(gpos::HashValue(&ptdi->subid), + gpos::HashValue(&ptdi->ocolid))); +} + +// equality function +BOOL +TableDescIdent::EqualFunc(const TableDescIdent *pltdi, + const TableDescIdent *prtdi) +{ + GPOS_ASSERT(nullptr != pltdi && nullptr != prtdi); + return pltdi->pmdid->Equals(prtdi->pmdid) && pltdi->subid == prtdi->subid && + pltdi->ocolid == prtdi->ocolid; +} + + +// expr is CLogicalGet,CLogicalDynamicGet or CLogicalCTEConsumer ? +BOOL +CDedupSupersetPreprocessor::PexprIsLogicalOutput(CExpression *pexpr) +{ + GPOS_ASSERT(pexpr); + COperator *pop = pexpr->Pop(); + return pop->Eopid() == COperator::EopLogicalGet || + pop->Eopid() == COperator::EopLogicalDynamicGet || + pop->Eopid() == COperator::EopLogicalCTEConsumer; +} + +// Get the meta id from expression +// The expression must be CLogicalGet/CLogicalDynamicGet/CLogicalCTEConsumer +// for the CLogicalGet/CLogicalDynamicGet, we can direct get the meta id by table desc. +// But for the CLogicalCTEConsumer, we will create a meta id from CTEid +IMDId * +CDedupSupersetPreprocessor::PexprGetIMDid(CMemoryPool *mp, CExpression *pexpr) +{ + GPOS_ASSERT(pexpr); + COperator *poper = pexpr->Pop(); + IMDId *pmdid = nullptr; + switch (poper->Eopid()) + { + case COperator::EopLogicalGet: + { + CLogicalGet *pop_get = CLogicalGet::PopConvert(poper); + if (pop_get->Ptabdesc()) + { + pmdid = pop_get->Ptabdesc()->MDId()->Copy(mp); + } + break; + } + case COperator::EopLogicalDynamicGet: + { + CLogicalDynamicGet *pop_dynamicget = + CLogicalDynamicGet::PopConvert(poper); + if (pop_dynamicget->Ptabdesc()) + { + pmdid = pop_dynamicget->Ptabdesc()->MDId()->Copy(mp); + } + break; + } + case COperator::EopLogicalCTEConsumer: + { + CLogicalCTEConsumer *pop_cc = + CLogicalCTEConsumer::PopConvert(poper); + pmdid = GPOS_NEW(mp) CMDIdGPDB( + IMDId::EmdidGeneral, pop_cc->UlCTEId() + CTEID_TO_OID_START); + break; + } + default: + break; + } + + return pmdid; +} + +// full the maps with superset expression +void +CDedupSupersetPreprocessor::ChildExprFullSuperset(CMemoryPool *mp, + CExpression *pexpr, + TDIToUlongPtrMap *eqcrmaps, + BOOL *dedupulmasks, ULONG ul) +{ + CScalarSubqueryAny *pop_subany = + CScalarSubqueryAny::PopConvert(pexpr->Pop()); + CExpression *pexpr_child0 = (*pexpr)[0]; + IMDId *tdimid = PexprGetIMDid(mp, pexpr_child0); + CScalarIdent *pop_ident = CScalarIdent::PopConvert((*pexpr)[1]->Pop()); + ULONG ulidx_subany; + + if (nullptr == tdimid) + { + // skip + return; + } + CColRefSet *colrefsets = pexpr_child0->DeriveOutputColumns(); + + // The colref(in CScalarSubqueryAny) must in the output + (void) colrefsets->ExtractIndex(pop_subany->Pcr(), &ulidx_subany); + GPOS_ASSERT(ulidx_subany != gpos::ulong_max); + + TableDescIdent tdikey = + TableDescIdent(tdimid, ulidx_subany, pop_ident->Pcr()->Id()); + ULONG *pul = eqcrmaps->Find(&tdikey); + if (!pul) + { + // new find superset, register it + eqcrmaps->Insert(GPOS_NEW(mp) TableDescIdent(tdimid, ulidx_subany, + pop_ident->Pcr()->Id()), + GPOS_NEW(mp) ULONG(ul)); + } + else + { + // already register the same superset expr, direct remove it + // + // ex. + // select * from t1,t2 where + // t1.v1 in (select v3 from t2) <- colref(v3) + // and + // t1.v1 in (select v3 from t2); <- same colref(v3) but id is different + dedupulmasks[ul] = false; + } +} + +// full the dedups array with subset(simply logicalget) +void +CDedupSupersetPreprocessor::ChildExprFullSimplySubset( + CMemoryPool *mp, CExpression *pexpr, TDIToUlongPtrMap *eqcrmaps, + BOOL *dedupulmasks) +{ + CScalarSubqueryAny *pop_subany = + CScalarSubqueryAny::PopConvert(pexpr->Pop()); + CExpression *pexpr_subchild0 = (*(*pexpr)[0])[0]; + IMDId *tdimid = PexprGetIMDid(mp, pexpr_subchild0); + CScalarIdent *pop_ident = CScalarIdent::PopConvert((*pexpr)[1]->Pop()); + ULONG ulidx_subany; + GPOS_ASSERT(PexprIsLogicalOutput(pexpr_subchild0)); + + if (nullptr == tdimid) + { + // skip current subset + return; + } + + CColRefSet *colrefsets = pexpr_subchild0->DeriveOutputColumns(); + // The colref(in CScalarSubqueryAny) must in the output + (void) colrefsets->ExtractIndex(pop_subany->Pcr(), &ulidx_subany); + GPOS_ASSERT(ulidx_subany != gpos::ulong_max); + + TableDescIdent tdikey = + TableDescIdent(tdimid, ulidx_subany, pop_ident->Pcr()->Id()); + ULONG *pul = eqcrmaps->Find(&tdikey); + if (pul) + { + // remove the expression in ul + dedupulmasks[*pul] = false; + } +} + +// Extract the sub child from the join(inner join) expr +// +// |--CLogicalNAryJoin +// | |--CLogicalGet "t2" || CLogicalDynamicGet "t2" || CLogicalCTEConsumer (0) <- extract the expression if colref belong child0 +// | |--CLogicalGet "t3" || CLogicalDynamicGet "t3" || CLogicalCTEConsumer (1) <- extract the expression if colref belong child1 +// | +--Scalar +CExpression * +CDedupSupersetPreprocessor::PexprExtractSubChildJoinExpr( + CExpression *pexpr, const CColRef *subany_col) +{ + GPOS_ASSERT(pexpr); + COperator *poper = pexpr->Pop(); + CExpression *child_pexpr = nullptr; + + if (poper->Eopid() == COperator::EopLogicalNAryJoin) + { + GPOS_ASSERT(pexpr->Arity() >= 2); + CExpression *pexpr_child0 = (*pexpr)[0]; + CExpression *pexpr_child1 = (*pexpr)[1]; + + if (PexprIsLogicalOutput(pexpr_child0) && + pexpr_child0->DeriveOutputColumns()->FMember(subany_col)) + { + child_pexpr = pexpr_child0; + } + else if (PexprIsLogicalOutput(pexpr_child1) && + pexpr_child1->DeriveOutputColumns()->FMember(subany_col)) + { + child_pexpr = pexpr_child1; + } + } + + return child_pexpr; +} + +ULONG * +CDedupSupersetPreprocessor::DriveFullJoinSubset(CMemoryPool *mp, + TDIToUlongPtrMap *eqcrmaps, + CExpression *pexpr_join, + const CColRef *pcr_sbany, + const CColRef *pcr_ident) +{ + CExpression *pexpr_join_child; + IMDId *tdimid = nullptr; + ULONG ulidx_subany; + + pexpr_join_child = PexprExtractSubChildJoinExpr(pexpr_join, pcr_sbany); + + if (nullptr == pexpr_join_child) + { + return nullptr; + } + + GPOS_ASSERT(PexprIsLogicalOutput(pexpr_join_child)); + tdimid = PexprGetIMDid(mp, pexpr_join_child); + if (nullptr == tdimid) + { + return nullptr; + } + + CColRefSet *colrefsets = pexpr_join_child->DeriveOutputColumns(); + // The colref(in CScalarSubqueryAny) must in the output + (void) colrefsets->ExtractIndex(pcr_sbany, &ulidx_subany); + GPOS_ASSERT(ulidx_subany != gpos::ulong_max); + + TableDescIdent tdikey = + TableDescIdent(tdimid, ulidx_subany, pcr_ident->Id()); + + return eqcrmaps->Find(&tdikey); +} + +// full the dedups array with subset(inner join) +void +CDedupSupersetPreprocessor::ChildExprFullJoinSubset(CMemoryPool *mp, + CExpression *pexpr, + TDIToUlongPtrMap *eqcrmaps, + BOOL *dedupulmasks) +{ + CScalarSubqueryAny *pop_subany = + CScalarSubqueryAny::PopConvert(pexpr->Pop()); + CExpression *pexpr_join = (*pexpr)[0]; + CScalarIdent *pop_ident = CScalarIdent::PopConvert((*pexpr)[1]->Pop()); + + ULONG *pul = DriveFullJoinSubset(mp, eqcrmaps, pexpr_join, + pop_subany->Pcr(), pop_ident->Pcr()); + if (pul) + { + // remove the expression in ul + dedupulmasks[*pul] = false; + return; + } + + // not match, we can try the join key. + // ex. + // t1(v1 int, v2 int) + // t2(v1 int, v2 int) + // t3(v1 int, v2 int) + // + // select v1 from t1 where v1 in (select t2.v1 from t2) and v1 in (select t3.v1 from t2,t3 where t3.v1 = t2.v1); + // + // In this case, we still can use the v3.v1 as the subset key + if (pexpr_join->Arity() == 3 && + (*pexpr_join)[2]->Pop()->Eopid() == COperator::EopScalarCmp) + { + CExpression *pexpr_cmp = (*pexpr_join)[2]; + CScalarCmp *pop_cmp = CScalarCmp::PopConvert(pexpr_cmp->Pop()); + if (pop_cmp->ParseCmpType() != IMDType::EcmptEq || + pexpr_cmp->Arity() != 2 || + (*pexpr_cmp)[0]->Pop()->Eopid() != COperator::EopScalarIdent || + (*pexpr_cmp)[1]->Pop()->Eopid() != COperator::EopScalarIdent) + { + // do nothing + return; + } + CScalarIdent *pop_id1 = + CScalarIdent::PopConvert((*pexpr_cmp)[0]->Pop()); + CScalarIdent *pop_id2 = + CScalarIdent::PopConvert((*pexpr_cmp)[1]->Pop()); + ULONG *pul = nullptr; + + if (pop_id1->Pcr() == pop_subany->Pcr()) + { + pul = DriveFullJoinSubset(mp, eqcrmaps, pexpr_join, pop_id2->Pcr(), + pop_ident->Pcr()); + } + else if (pop_id2->Pcr() == pop_subany->Pcr()) + { + pul = DriveFullJoinSubset(mp, eqcrmaps, pexpr_join, pop_id1->Pcr(), + pop_ident->Pcr()); + } + + if (pul) + { + // remove the expression in ul + dedupulmasks[*pul] = false; + } + } +} + +// child expr in CScalarBoolOp(ANDOP) is the superset +// +// |--CScalarSubqueryAny(=)["v3" (20)] ... <- it's a superset +// |--CLogicalGet "t2"/CLogicalDynamicGet "t2"/CLogicalCTEConsumer (0) <- all fine +// +--CScalarIdent "v1" ... <- get the output +BOOL +CDedupSupersetPreprocessor::ChildExprIsSuperset(CExpression *pexpr) +{ + return pexpr->Pop()->Eopid() == COperator::EopScalarSubqueryAny && + pexpr->Arity() == 2 && PexprIsLogicalOutput((*pexpr)[0]) && + (*pexpr)[1]->Pop()->Eopid() == COperator::EopScalarIdent; +} + +// child expr in CScalarBoolOp(ANDOP) is the simply subset +// +// --CScalarSubqueryAny(=)["v3" (30)] <- it's a subset +// |--CLogicalSelect +// | |--CLogicalGet "t2" || CLogicalDynamicGet "t2" || CLogicalCTEConsumer (0) <- all fine +// | +--CScalar <- ANY scalar +// +--CScalarIdent "v1" (0) +BOOL +CDedupSupersetPreprocessor::ChildExprIsSimplySubset(CExpression *pexpr) +{ + GPOS_ASSERT(pexpr); + return pexpr->Pop()->Eopid() == COperator::EopScalarSubqueryAny && + pexpr->Arity() == 2 && + (*pexpr)[0]->Pop()->Eopid() == COperator::EopLogicalSelect && + (*pexpr)[1]->Pop()->Eopid() == COperator::EopScalarIdent && + (*pexpr)[0]->Arity() > 0 && PexprIsLogicalOutput((*(*pexpr)[0])[0]); +} + +// child expr in CScalarBoolOp(ANDOP) is the inner-join subset +// left-semi-join won't occur in the pre-process +// +// +--CScalarSubqueryAny(=)["v3" (20)] <- it's a subset +// |--CLogicalNAryJoin <- inner or smei join. Although semi join(EopLogicalLeftSemiJoin) is supported, but no semi join will occur at pre-process. +// | |--CLogicalGet "t2" || CLogicalDynamicGet "t2" || CLogicalCTEConsumer (0) <- all fine +// | |--CLogicalGet "t3" ("t3") +// | +--CScalarCmp (=) <- ANY scalar(won't be full/cross join condition) +// | |--CScalarIdent "v4" (21) +// | +--CScalarIdent "v6" (31) +// +--CScalarIdent "v1" (0) +BOOL +CDedupSupersetPreprocessor::ChildExprIsJoinSubset(CExpression *pexpr) +{ + GPOS_ASSERT(pexpr); + return pexpr->Pop()->Eopid() == COperator::EopScalarSubqueryAny && + pexpr->Arity() == 2 && + (*pexpr)[0]->Pop()->Eopid() == + COperator::EopLogicalNAryJoin && // inner join + (*pexpr)[1]->Pop()->Eopid() == COperator::EopScalarIdent && + (*pexpr)[0]->Arity() >= 2 && // at least left and right + (PexprIsLogicalOutput((*(*pexpr)[0])[0]) || + PexprIsLogicalOutput((*(*pexpr)[0])[1])); +} + +// mian driver +// If the same key in the current scalar has the following conditions: +// - select a superset in {table A} +// - select a subset in {table A} +// - the subset can be the get/dynamicget/cteconsumer or a inner join +// +// Then the scalar(superset in {table A}) can be removed. +// +// Example, for the schema: t1(v1, v2), t2(v3, v4) +// +// The query: +// select * from t1,t2 where t1.v1 in (select v3 from t2) and t1.v1 in (select v3 from t2 where v3 < 100); +// equivalent to: +// select * from t1,t2 where t1.v1 t1.v1 in (select v3 from t2 where v3 < 100); +// +// Similarly, CTE also can removed duplicate scalar +// with cte_t as (...) +// select * from t1 where v1 in (select v2 from cte_1) and v1 in (select v2 from cte_1 where {any condition}); +// equivalent to: +// select * from t where a in (select count(i) from s group by j); +// +// Also it also satisfies the cross join condition. +// select * from t1,t2 where t1.v1 in (select v2 from t2) and t1.v1 in (select v2 from t2 where v2 < 100); +// equivalent to: +// select * from t1,t2 where t1.v1 in (select v2 from t2 where v2 < 100); +// +// And the inner join can be the subset. +// select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6); +// equivalent to: +// select * from t1 where v1 v1 in (select v3 from t2,t3 where v4=v6); +CExpression * +CDedupSupersetPreprocessor::PexprPreprocess(CMemoryPool *mp, CExpression *pexpr) +{ + GPOS_CHECK_STACK_SIZE; + GPOS_ASSERT(nullptr != mp); + GPOS_ASSERT(nullptr != pexpr); + + // operator, possibly altered below if we need to change the operator + COperator *pop = pexpr->Pop(); + + if (pop->Eopid() == COperator::EopScalarBoolOp) + { + const ULONG arity = pexpr->Arity(); + BOOL *ulmasks = GPOS_NEW_ARRAY(mp, BOOL, arity); + BOOL *dedupulmasks = GPOS_NEW_ARRAY(mp, BOOL, arity); + clib::Memset(ulmasks, true, arity * sizeof(BOOL)); + clib::Memset(dedupulmasks, true, arity * sizeof(BOOL)); + + // TableDescIdent(table desc mdid, subany index, ident colid) -> ULONG(index of current ) + TDIToUlongPtrMap *eqcrmaps = GPOS_NEW(mp) TDIToUlongPtrMap(mp); + + if (arity < 2 || CScalarBoolOp::PopConvert(pop)->Eboolop() != + CScalarBoolOp::EboolopAnd) + { + goto not_match; + } + + for (ULONG ul = 0; ul < arity; ul++) + { + CExpression *child_pexpr = (*pexpr)[ul]; + if (ChildExprIsSuperset(child_pexpr)) + { + ChildExprFullSuperset(mp, child_pexpr, eqcrmaps, dedupulmasks, + ul); + // no need check this expression in next loop + ulmasks[ul] = false; + } + } + + // not found any superset expression + if (eqcrmaps->Size() == 0) + { + goto not_match; + } + + for (ULONG ul = 0; ul < arity; ul++) + { + // current is superset + if (!ulmasks[ul]) + { + continue; + } + + CExpression *child_pexpr = (*pexpr)[ul]; + + if (ChildExprIsSimplySubset(child_pexpr)) + { + ChildExprFullSimplySubset(mp, child_pexpr, eqcrmaps, + dedupulmasks); + continue; + } + + if (ChildExprIsJoinSubset(child_pexpr)) + { + ChildExprFullJoinSubset(mp, child_pexpr, eqcrmaps, + dedupulmasks); + } + } + + not_match: + CExpressionArray *pdrgpexprChildren = GPOS_NEW(mp) CExpressionArray(mp); + + for (ULONG ul = 0; ul < arity; ul++) + { + if (!dedupulmasks[ul]) + { + continue; + } + + (*pexpr)[ul]->AddRef(); + pdrgpexprChildren->Append((*pexpr)[ul]); + } + + pop->AddRef(); + + GPOS_DELETE_ARRAY(ulmasks); + GPOS_DELETE_ARRAY(dedupulmasks); + eqcrmaps->Release(); + + return GPOS_NEW(mp) CExpression(mp, pop, pdrgpexprChildren); + } + // FIXME: should we consider remove the CTE consumer from CTEINFO? + + const ULONG arity = pexpr->Arity(); + CExpressionArray *pdrgpexprChildren = GPOS_NEW(mp) CExpressionArray(mp); + + for (ULONG ul = 0; ul < arity; ul++) + { + CExpression *pexprChild = PexprPreprocess(mp, (*pexpr)[ul]); + + pdrgpexprChildren->Append(pexprChild); + } + + pop->AddRef(); + return GPOS_NEW(mp) CExpression(mp, pop, pdrgpexprChildren); +} \ No newline at end of file diff --git a/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp b/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp index 1e3496e54e0..e1c46c9fc8e 100644 --- a/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp +++ b/src/backend/gporca/libgpopt/src/operators/CExpressionPreprocessor.cpp @@ -24,6 +24,7 @@ #include "gpopt/base/CUtils.h" #include "gpopt/exception.h" #include "gpopt/mdcache/CMDAccessor.h" +#include "gpopt/operators/CDedupSupersetPreprocessor.h" #include "gpopt/operators/CExpressionFactorizer.h" #include "gpopt/operators/CExpressionUtils.h" #include "gpopt/operators/CJoinOrderHintsPreprocessor.h" @@ -3365,13 +3366,21 @@ CExpressionPreprocessor::PexprPreprocess( GPOS_CHECK_ABORT; pexprSimplifiedLimit->Release(); + // remove dedup superset scalar + CExpression *pexprRmDupSupersetScalar = + CDedupSupersetPreprocessor::PexprPreprocess(mp, pexprSimplifiedDistinct); + TRCAE_PREPROCESS_STEP(pexprRmDupSupersetScalar, + "Remove the dedup the superset"); + GPOS_CHECK_ABORT; + pexprSimplifiedDistinct->Release(); + // trim unnecessary existential subqueries CExpression *pexprTrimmed = - PexprTrimExistentialSubqueries(mp, pexprSimplifiedDistinct); + PexprTrimExistentialSubqueries(mp, pexprRmDupSupersetScalar); TRCAE_PREPROCESS_STEP(pexprTrimmed, "Trim unnecessary existential subqueries"); GPOS_CHECK_ABORT; - pexprSimplifiedDistinct->Release(); + pexprRmDupSupersetScalar->Release(); // collapse cascaded union / union all CExpression *pexprNaryUnionUnionAll = diff --git a/src/backend/gporca/libgpopt/src/operators/Makefile b/src/backend/gporca/libgpopt/src/operators/Makefile index c495244b032..5030aec79a0 100644 --- a/src/backend/gporca/libgpopt/src/operators/Makefile +++ b/src/backend/gporca/libgpopt/src/operators/Makefile @@ -189,7 +189,8 @@ OBJS = CExpression.o \ CStrictHashedDistributions.o \ COrderedAggPreprocessor.o \ CLeftJoinPruningPreprocessor.o \ - CJoinOrderHintsPreprocessor.o + CJoinOrderHintsPreprocessor.o \ + CDedupSupersetPreprocessor.o include $(top_srcdir)/src/backend/common.mk diff --git a/src/test/regress/expected/cte_prune.out b/src/test/regress/expected/cte_prune.out index 6dd8a640c9b..1b54f443487 100644 --- a/src/test/regress/expected/cte_prune.out +++ b/src/test/regress/expected/cte_prune.out @@ -918,7 +918,7 @@ explain verbose with frequent_ss_items as and d_year in (1999,1999+1,1999+2,1999+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4) -select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) +select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items where true) and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); QUERY PLAN -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- @@ -1019,7 +1019,7 @@ explain verbose with ws_wh as from tpcds_web_sales ws1,tpcds_web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); +select * from t1 where t1.v1 in (select ws_order_number from ws_wh where true) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=2952.11..2952.33 rows=14 width=12) @@ -1100,7 +1100,7 @@ explain verbose with ws_wh as from tpcds_web_sales ws1,tpcds_web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); +select * from t1 where t1.v1 in (select wh1 from ws_wh where true) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); QUERY PLAN ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=2952.68..2952.91 rows=14 width=12) diff --git a/src/test/regress/expected/cte_prune_optimizer.out b/src/test/regress/expected/cte_prune_optimizer.out index bd8f569f691..ed3516845b2 100644 --- a/src/test/regress/expected/cte_prune_optimizer.out +++ b/src/test/regress/expected/cte_prune_optimizer.out @@ -1077,7 +1077,7 @@ explain verbose with frequent_ss_items as and d_year in (1999,1999+1,1999+2,1999+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4) -select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) +select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items where true) and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------- @@ -1144,7 +1144,7 @@ explain verbose with ws_wh as from tpcds_web_sales ws1,tpcds_web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); +select * from t1 where t1.v1 in (select ws_order_number from ws_wh where true) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1730.00 rows=1 width=12) @@ -1198,7 +1198,7 @@ explain verbose with ws_wh as from tpcds_web_sales ws1,tpcds_web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); +select * from t1 where t1.v1 in (select wh1 from ws_wh where true) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------- Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1730.00 rows=1 width=12) diff --git a/src/test/regress/expected/dedupset.out b/src/test/regress/expected/dedupset.out new file mode 100644 index 00000000000..bcf9b7f1123 --- /dev/null +++ b/src/test/regress/expected/dedupset.out @@ -0,0 +1,1936 @@ +SET optimizer_trace_fallback = on; +-- start_ignore +drop table if exists t1; +drop table if exists t2; +drop table if exists t3; +drop table if exists pt1; +drop table if exists pt2; +-- end_ignore +create table t1(v1 int, v2 int); +create table t2(v3 int, v4 int); +create table t3(v5 int, v6 int); +insert into t1 values(generate_series(1, 100), generate_series(1, 100)); +insert into t2 values(generate_series(1, 100), generate_series(1, 100)); +insert into t3 values(generate_series(1, 100), generate_series(1, 100)); +CREATE TABLE pt1 ( + v1 INT, + v2 INT +) PARTITION BY RANGE (v1); +CREATE TABLE pt1_p1 PARTITION OF pt1 FOR VALUES FROM (0) TO (20); +CREATE TABLE pt1_p2 PARTITION OF pt1 FOR VALUES FROM (20) TO (40); +CREATE TABLE pt1_p3 PARTITION OF pt1 FOR VALUES FROM (40) TO (60); +CREATE TABLE pt1_p4 PARTITION OF pt1 FOR VALUES FROM (60) TO (400); +CREATE TABLE pt2 ( + v3 INT, + v4 INT +) PARTITION BY RANGE (v3); +CREATE TABLE pt2_p1 PARTITION OF pt2 FOR VALUES FROM (0) TO (20); +CREATE TABLE pt2_p2 PARTITION OF pt2 FOR VALUES FROM (20) TO (40); +CREATE TABLE pt2_p3 PARTITION OF pt2 FOR VALUES FROM (40) TO (60); +CREATE TABLE pt2_p4 PARTITION OF pt2 FOR VALUES FROM (60) TO (400); +insert into pt1 values(generate_series(1, 100), generate_series(1, 100)); +insert into pt2 values(generate_series(1, 100), generate_series(1, 100)); +analyze t1; +analyze t2; +analyze t3; +analyze pt1; +analyze pt2; +-- dedup the subquery with projection +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.90..4.39 rows=3 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=2.90..4.35 rows=1 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=12) + Output: t1.v1, t1.v2, t2.v3 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(23 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10) and v1 in (select v3 from t2); -- change the order + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.90..4.39 rows=3 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=2.90..4.35 rows=1 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=12) + Output: t1.v1, t1.v2, t2.v3 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(23 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +---------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=1.45..2.93 rows=3 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(15 rows) + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 1 | 1 + 5 | 5 + 6 | 6 + 9 | 9 +(9 rows) + +select * from t1 where v1 in (select v3 from t2 where v3 < 10); + v1 | v2 +----+---- + 1 | 1 + 5 | 5 + 6 | 6 + 9 | 9 + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 +(9 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2 where v3 < 10); + QUERY PLAN +------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=2.23..3.50 rows=8 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=2.23..3.39 rows=3 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2_1.v3) + -> Hash Semi Join (cost=1.11..2.23 rows=3 width=12) + Output: pt1.v1, pt1.v2, pt2.v3 + Hash Cond: (pt1.v1 = pt2.v3) + -> Seq Scan on public.pt1_p1 pt1 (cost=0.00..1.08 rows=3 width=8) + Output: pt1.v1, pt1.v2 + Filter: (pt1.v1 < 10) + -> Hash (cost=1.08..1.08 rows=3 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..1.08 rows=3 width=4) + Output: pt2.v3 + -> Seq Scan on public.pt2_p1 pt2 (cost=0.00..1.08 rows=3 width=4) + Output: pt2.v3 + Filter: (pt2.v3 < 10) + -> Hash (cost=1.08..1.08 rows=3 width=4) + Output: pt2_1.v3 + -> Partition Selector (selector id: $1) (cost=0.00..1.08 rows=3 width=4) + Output: pt2_1.v3 + -> Seq Scan on public.pt2_p1 pt2_1 (cost=0.00..1.08 rows=3 width=4) + Output: pt2_1.v3 + Filter: (pt2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(27 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2 where v3 < 10); + QUERY PLAN +------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=1.11..2.35 rows=8 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=1.11..2.23 rows=3 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Seq Scan on public.pt1_p1 pt1 (cost=0.00..1.08 rows=3 width=8) + Output: pt1.v1, pt1.v2 + Filter: (pt1.v1 < 10) + -> Hash (cost=1.08..1.08 rows=3 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..1.08 rows=3 width=4) + Output: pt2.v3 + -> Seq Scan on public.pt2_p1 pt2 (cost=0.00..1.08 rows=3 width=4) + Output: pt2.v3 + Filter: (pt2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(17 rows) + +select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2 where v3 < 10); + v1 | v2 +----+---- + 1 | 1 + 5 | 5 + 6 | 6 + 9 | 9 + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 +(9 rows) + +select * from pt1 where v1 in (select v3 from pt2 where v3 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 5 | 5 + 6 | 6 + 9 | 9 + 1 | 1 +(9 rows) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=10000000015.48..10000000015.49 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=10000000015.43..10000000015.48 rows=3 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=10000000015.43..10000000015.44 rows=1 width=8) + Output: PARTIAL count(*) + -> Nested Loop (cost=10000000004.85..10000000014.75 rows=270 width=0) + -> Result (cost=4.85..4.86 rows=3 width=0) + -> Unique (cost=4.85..4.86 rows=3 width=0) + Output: (RowIdExpr) + Group Key: (RowIdExpr) + -> Sort (cost=4.85..4.86 rows=3 width=0) + Output: (RowIdExpr) + Sort Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.96..4.83 rows=3 width=0) + Output: (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=2.96..4.78 rows=3 width=0) + Output: (RowIdExpr) + Hash Cond: (t2_1.v3 = t1.v1) + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Hash (cost=2.93..2.93 rows=3 width=8) + Output: t1.v1, t2_2.v3, (RowIdExpr) + -> Hash Semi Join (cost=1.48..2.93 rows=3 width=8) + Output: t1.v1, t2_2.v3, RowIdExpr + Hash Cond: (t1.v1 = t2_2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=1.44..1.44 rows=3 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.42 rows=3 width=4) + Output: t2_2.v3 + Filter: (t2_2.v3 < 10) + -> Materialize (cost=0.00..3.17 rows=100 width=0) + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=0) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=0) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(39 rows) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1 where v3 < 10); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=10000000013.55..10000000013.56 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=10000000013.49..10000000013.54 rows=3 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=10000000013.49..10000000013.50 rows=1 width=8) + Output: PARTIAL count(*) + -> Nested Loop (cost=10000000001.48..10000000012.81 rows=270 width=0) + -> Hash Semi Join (cost=1.48..2.93 rows=3 width=0) + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=1.44..1.44 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + -> Materialize (cost=0.00..3.17 rows=100 width=0) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=0) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=0) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(21 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + count +------- + 900 +(1 row) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1 where v3 < 10); + count +------- + 900 +(1 row) + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + QUERY PLAN +-------------------------------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=10000000015.48..10000000015.49 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=10000000015.43..10000000015.48 rows=3 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=10000000015.43..10000000015.44 rows=1 width=8) + Output: PARTIAL count(*) + -> Nested Loop (cost=10000000004.85..10000000014.75 rows=270 width=0) + -> Result (cost=4.85..4.86 rows=3 width=0) + -> Unique (cost=4.85..4.86 rows=3 width=0) + Output: (RowIdExpr) + Group Key: (RowIdExpr) + -> Sort (cost=4.85..4.86 rows=3 width=0) + Output: (RowIdExpr) + Sort Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.96..4.83 rows=3 width=0) + Output: (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=2.96..4.78 rows=3 width=0) + Output: (RowIdExpr) + Hash Cond: (t2_1.v3 = t1.v1) + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Hash (cost=2.93..2.93 rows=3 width=8) + Output: t1.v1, t2_2.v3, (RowIdExpr) + -> Hash Semi Join (cost=1.48..2.93 rows=3 width=8) + Output: t1.v1, t2_2.v3, RowIdExpr + Hash Cond: (t1.v1 = t2_2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=1.44..1.44 rows=3 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.42 rows=3 width=4) + Output: t2_2.v3 + Filter: (t2_2.v3 < 10) + -> Materialize (cost=0.00..3.17 rows=100 width=0) + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=0) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=0) + Settings: enable_parallel = 'off', optimizer = 'off', optimizer_cte_inlining_bound = '2' + Optimizer: Postgres query optimizer +(39 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + count +------- + 900 +(1 row) + +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +--------------------------------------------------------------------------------------- + Aggregate (cost=4.40..4.41 rows=1 width=8) + Output: sum(t1.v1) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=2.90..4.39 rows=3 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=2.90..4.35 rows=1 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=8) + Output: t1.v1, t2.v3 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=4) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(25 rows) + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +---------------------------------------------------------------------------------- + Aggregate (cost=2.94..2.95 rows=1 width=8) + Output: sum(t1.v1) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=1.45..2.93 rows=3 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=4) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(17 rows) + +select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + sum +----- + 45 +(1 row) + +select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10); + sum +----- + 45 +(1 row) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) order by v1; + QUERY PLAN +--------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=4.36..4.41 rows=3 width=8) + Output: t1.v1, t1.v2 + Merge Key: t1.v1 + -> Sort (cost=4.36..4.36 rows=1 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v1 + -> Hash Semi Join (cost=2.90..4.35 rows=1 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=12) + Output: t1.v1, t1.v2, t2.v3 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(27 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10) order by v1; + QUERY PLAN +--------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.90..2.94 rows=3 width=8) + Output: t1.v1, t1.v2 + Merge Key: t1.v1 + -> Sort (cost=2.90..2.90 rows=1 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v1 + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(19 rows) + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) order by v1; + v1 | v2 +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 +(9 rows) + +select * from t1 where v1 in (select v3 from t2 where v3 < 10) order by v1; + v1 | v2 +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 +(9 rows) + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) group by v2; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=4.38..4.45 rows=3 width=12) + Output: (sum(t1.v1)), t1.v2 + -> GroupAggregate (cost=4.38..4.40 rows=1 width=12) + Output: sum(t1.v1), t1.v2 + Group Key: t1.v2 + -> Sort (cost=4.38..4.39 rows=1 width=8) + Output: t1.v2, t1.v1 + Sort Key: t1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.90..4.37 rows=1 width=8) + Output: t1.v2, t1.v1 + Hash Key: t1.v2 + -> Hash Semi Join (cost=2.90..4.35 rows=1 width=8) + Output: t1.v2, t1.v1 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=12) + Output: t1.v1, t1.v2, t2.v3 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(32 rows) + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10) group by v2; + QUERY PLAN +---------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=2.92..2.98 rows=3 width=12) + Output: (sum(t1.v1)), t1.v2 + -> GroupAggregate (cost=2.92..2.94 rows=1 width=12) + Output: sum(t1.v1), t1.v2 + Group Key: t1.v2 + -> Sort (cost=2.92..2.92 rows=1 width=8) + Output: t1.v2, t1.v1 + Sort Key: t1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.45..2.91 rows=1 width=8) + Output: t1.v2, t1.v1 + Hash Key: t1.v2 + -> Hash Semi Join (cost=1.45..2.89 rows=1 width=8) + Output: t1.v2, t1.v1 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v1 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.42 rows=3 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(24 rows) + +select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) group by v2; + sum +----- + 1 + 5 + 6 + 9 + 2 + 3 + 4 + 7 + 8 +(9 rows) + +select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10) group by v2; + sum +----- + 2 + 3 + 4 + 7 + 8 + 1 + 5 + 6 + 9 +(9 rows) + +-- dedup the same subqueryany +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2); + QUERY PLAN +---------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=3.50..7.08 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=3.50..5.75 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=1.75..3.54 rows=33 width=12) + Output: t1.v1, t1.v2, t2.v3 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(20 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2); + QUERY PLAN +------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=1.75..4.87 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=1.75..3.54 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(13 rows) + +select count(*) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2); + count +------- + 100 +(1 row) + +select count(*) from t1 where v1 in (select v3 from t2); + count +------- + 100 +(1 row) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2); + QUERY PLAN +-------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=9.83..16.58 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=9.83..15.25 rows=33 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2_1.v3) + -> Hash Semi Join (cost=4.92..9.87 rows=33 width=12) + Output: pt1.v1, pt1.v2, pt2.v3 + Hash Cond: (pt1.v1 = pt2.v3) + -> Append (cost=0.00..4.50 rows=33 width=8) + Partition Selectors: $0, $1 + -> Seq Scan on public.pt1_p1 pt1_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt1_1.v1, pt1_1.v2 + -> Seq Scan on public.pt1_p2 pt1_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_2.v1, pt1_2.v2 + -> Seq Scan on public.pt1_p3 pt1_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_3.v1, pt1_3.v2 + -> Seq Scan on public.pt1_p4 pt1_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt1_4.v1, pt1_4.v2 + -> Hash (cost=4.50..4.50 rows=33 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..4.50 rows=33 width=4) + Output: pt2.v3 + -> Append (cost=0.00..4.50 rows=33 width=4) + -> Seq Scan on public.pt2_p1 pt2_2 (cost=0.00..1.06 rows=6 width=4) + Output: pt2_2.v3 + -> Seq Scan on public.pt2_p2 pt2_3 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_3.v3 + -> Seq Scan on public.pt2_p3 pt2_4 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_4.v3 + -> Seq Scan on public.pt2_p4 pt2_5 (cost=0.00..1.14 rows=14 width=4) + Output: pt2_5.v3 + -> Hash (cost=4.50..4.50 rows=33 width=4) + Output: pt2_1.v3 + -> Partition Selector (selector id: $1) (cost=0.00..4.50 rows=33 width=4) + Output: pt2_1.v3 + -> Append (cost=0.00..4.50 rows=33 width=4) + -> Seq Scan on public.pt2_p1 pt2_6 (cost=0.00..1.06 rows=6 width=4) + Output: pt2_6.v3 + -> Seq Scan on public.pt2_p2 pt2_7 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_7.v3 + -> Seq Scan on public.pt2_p3 pt2_8 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_8.v3 + -> Seq Scan on public.pt2_p4 pt2_9 (cost=0.00..1.14 rows=14 width=4) + Output: pt2_9.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(46 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2); + QUERY PLAN +-------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=4.92..11.21 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=4.92..9.87 rows=33 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Append (cost=0.00..4.50 rows=33 width=8) + Partition Selectors: $0 + -> Seq Scan on public.pt1_p1 pt1_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt1_1.v1, pt1_1.v2 + -> Seq Scan on public.pt1_p2 pt1_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_2.v1, pt1_2.v2 + -> Seq Scan on public.pt1_p3 pt1_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_3.v1, pt1_3.v2 + -> Seq Scan on public.pt1_p4 pt1_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt1_4.v1, pt1_4.v2 + -> Hash (cost=4.50..4.50 rows=33 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..4.50 rows=33 width=4) + Output: pt2.v3 + -> Append (cost=0.00..4.50 rows=33 width=4) + -> Seq Scan on public.pt2_p1 pt2_1 (cost=0.00..1.06 rows=6 width=4) + Output: pt2_1.v3 + -> Seq Scan on public.pt2_p2 pt2_2 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_2.v3 + -> Seq Scan on public.pt2_p3 pt2_3 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_3.v3 + -> Seq Scan on public.pt2_p4 pt2_4 (cost=0.00..1.14 rows=14 width=4) + Output: pt2_4.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(30 rows) + +select count(*) from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2); + count +------- + 100 +(1 row) + +select count(*) from pt1 where v1 in (select v3 from pt2); + count +------- + 100 +(1 row) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=10000000137.03..10000000137.04 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=10000000136.97..10000000137.02 rows=3 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=10000000136.97..10000000136.98 rows=1 width=8) + Output: PARTIAL count(*) + -> Nested Loop (cost=10000000004.17..10000000128.64 rows=3333 width=0) + -> Hash Semi Join (cost=4.17..6.42 rows=33 width=0) + Hash Cond: (t1.v1 = t2_2.v3) + -> Hash Semi Join (cost=2.08..3.87 rows=33 width=8) + Output: t1.v1, t2_1.v3 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.33 rows=33 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=0) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=0) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=0) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(27 rows) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=10000000102.06..10000000102.07 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=10000000102.00..10000000102.05 rows=3 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=10000000102.00..10000000102.01 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=10000000002.08..10000000093.67 rows=3333 width=0) + Hash Cond: (t1.v1 = t2_1.v3) + -> Nested Loop (cost=10000000000.00..10000000045.75 rows=3333 width=4) + Output: t1.v1 + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=0) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=0) + -> Materialize (cost=0.00..1.50 rows=33 width=4) + Output: t1.v1 + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(22 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + count +------- + 10000 +(1 row) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1); + count +------- + 10000 +(1 row) + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + QUERY PLAN +--------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=10000000137.03..10000000137.04 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=10000000136.97..10000000137.02 rows=3 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=10000000136.97..10000000136.98 rows=1 width=8) + Output: PARTIAL count(*) + -> Nested Loop (cost=10000000004.17..10000000128.64 rows=3333 width=0) + -> Hash Semi Join (cost=4.17..6.42 rows=33 width=0) + Hash Cond: (t1.v1 = t2_2.v3) + -> Hash Semi Join (cost=2.08..3.87 rows=33 width=8) + Output: t1.v1, t2_1.v3 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.33 rows=33 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=0) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=0) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=0) + Settings: enable_parallel = 'off', optimizer = 'off', optimizer_cte_inlining_bound = '2' + Optimizer: Postgres query optimizer +(27 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + count +------- + 10000 +(1 row) + +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; +-- dedup the subquery with inner join +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=7.71..11.29 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=7.71..9.96 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Hash Semi Join (cost=5.96..7.75 rows=33 width=12) + Output: t1.v1, t1.v2, t2_1.v3 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=5.54..5.54 rows=33 width=4) + Output: t2_1.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.42..5.54 rows=33 width=4) + Output: t2_1.v3 + Hash Key: t2_1.v3 + -> Hash Join (cost=2.42..4.88 rows=33 width=4) + Output: t2_1.v3 + Hash Cond: (t2_1.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..2.00 rows=33 width=8) + Output: t2_1.v4, t2_1.v3 + Hash Key: t2_1.v4 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=8) + Output: t2_1.v4, t2_1.v3 + -> Hash (cost=2.00..2.00 rows=33 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..2.00 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(36 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t3,t2 where v4=v6); -- change the join order,different index in inner join + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=7.71..11.29 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=7.71..9.96 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Hash Semi Join (cost=5.96..7.75 rows=33 width=12) + Output: t1.v1, t1.v2, t2_1.v3 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=5.54..5.54 rows=33 width=4) + Output: t2_1.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.42..5.54 rows=33 width=4) + Output: t2_1.v3 + Hash Key: t2_1.v3 + -> Hash Join (cost=2.42..4.88 rows=33 width=4) + Output: t2_1.v3 + Hash Cond: (t3.v6 = t2_1.v4) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..2.00 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + -> Hash (cost=2.00..2.00 rows=33 width=8) + Output: t2_1.v4, t2_1.v3 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..2.00 rows=33 width=8) + Output: t2_1.v4, t2_1.v3 + Hash Key: t2_1.v4 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=8) + Output: t2_1.v4, t2_1.v3 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(36 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2,t3 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=5.96..9.08 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=5.96..7.75 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=5.54..5.54 rows=33 width=4) + Output: t2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.42..5.54 rows=33 width=4) + Output: t2.v3 + Hash Key: t2.v3 + -> Hash Join (cost=2.42..4.88 rows=33 width=4) + Output: t2.v3 + Hash Cond: (t2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..2.00 rows=33 width=8) + Output: t2.v4, t2.v3 + Hash Key: t2.v4 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=8) + Output: t2.v4, t2.v3 + -> Hash (cost=2.00..2.00 rows=33 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..2.00 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(29 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=6.04..6.14 rows=7 width=8) + Output: t1.v1, t1.v2 + -> Result (cost=6.04..6.05 rows=2 width=8) + Output: t1.v1, t1.v2 + -> Unique (cost=6.04..6.05 rows=2 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Group Key: (RowIdExpr) + -> Sort (cost=6.04..6.04 rows=2 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Sort Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=4.50..6.02 rows=2 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=4.50..5.98 rows=2 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Cond: (t2.v3 = t1.v1) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3, t2.v4 + -> Hash (cost=4.47..4.47 rows=2 width=12) + Output: t1.v1, t1.v2, t2_1.v3, (RowIdExpr) + -> Hash Semi Join (cost=3.03..4.47 rows=2 width=12) + Output: t1.v1, t1.v2, t2_1.v3, RowIdExpr + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=3.01..3.01 rows=1 width=4) + Output: t2_1.v3 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=1.50..3.01 rows=1 width=4) + Output: t2_1.v3 + Hash Key: t2_1.v3 + -> Hash Join (cost=1.50..2.99 rows=1 width=4) + Output: t2_1.v3 + Hash Cond: (t2_1.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..1.47 rows=3 width=8) + Output: t2_1.v4, t2_1.v3 + Hash Key: t2_1.v4 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=8) + Output: t2_1.v4, t2_1.v3 + Filter: (t2_1.v4 < 10) + -> Hash (cost=1.47..1.47 rows=3 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice5; segments: 3) (cost=0.00..1.47 rows=3 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.42 rows=3 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(49 rows) + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 9 | 9 + 8 | 8 + 6 | 6 + 7 | 7 + 1 | 1 + 5 | 5 +(9 rows) + +select * from t1 where v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 5 | 5 + 6 | 6 + 9 | 9 + 1 | 1 +(9 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=14.04..20.79 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=14.04..19.46 rows=33 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Hash Semi Join (cost=9.12..14.08 rows=33 width=12) + Output: pt1.v1, pt1.v2, pt2_1.v3 + Hash Cond: (pt1.v1 = pt2_1.v3) + -> Append (cost=0.00..4.50 rows=33 width=8) + Partition Selectors: $0, $1 + -> Seq Scan on public.pt1_p1 pt1_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt1_1.v1, pt1_1.v2 + -> Seq Scan on public.pt1_p2 pt1_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_2.v1, pt1_2.v2 + -> Seq Scan on public.pt1_p3 pt1_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_3.v1, pt1_3.v2 + -> Seq Scan on public.pt1_p4 pt1_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt1_4.v1, pt1_4.v2 + -> Hash (cost=8.71..8.71 rows=33 width=4) + Output: pt2_1.v3 + -> Partition Selector (selector id: $0) (cost=2.42..8.71 rows=33 width=4) + Output: pt2_1.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.42..8.71 rows=33 width=4) + Output: pt2_1.v3 + Hash Key: pt2_1.v3 + -> Hash Join (cost=2.42..8.04 rows=33 width=4) + Output: pt2_1.v3 + Hash Cond: (pt2_1.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..5.17 rows=33 width=8) + Output: pt2_1.v4, pt2_1.v3 + Hash Key: pt2_1.v4 + -> Append (cost=0.00..4.50 rows=33 width=8) + -> Seq Scan on public.pt2_p1 pt2_6 (cost=0.00..1.06 rows=6 width=8) + Output: pt2_6.v4, pt2_6.v3 + -> Seq Scan on public.pt2_p2 pt2_7 (cost=0.00..1.07 rows=7 width=8) + Output: pt2_7.v4, pt2_7.v3 + -> Seq Scan on public.pt2_p3 pt2_8 (cost=0.00..1.07 rows=7 width=8) + Output: pt2_8.v4, pt2_8.v3 + -> Seq Scan on public.pt2_p4 pt2_9 (cost=0.00..1.14 rows=14 width=8) + Output: pt2_9.v4, pt2_9.v3 + -> Hash (cost=2.00..2.00 rows=33 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..2.00 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + -> Hash (cost=4.50..4.50 rows=33 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $1) (cost=0.00..4.50 rows=33 width=4) + Output: pt2.v3 + -> Append (cost=0.00..4.50 rows=33 width=4) + -> Seq Scan on public.pt2_p1 pt2_2 (cost=0.00..1.06 rows=6 width=4) + Output: pt2_2.v3 + -> Seq Scan on public.pt2_p2 pt2_3 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_3.v3 + -> Seq Scan on public.pt2_p3 pt2_4 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_4.v3 + -> Seq Scan on public.pt2_p4 pt2_5 (cost=0.00..1.14 rows=14 width=4) + Output: pt2_5.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(62 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from t3,pt2 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=14.04..20.79 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=14.04..19.46 rows=33 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Hash Semi Join (cost=9.12..14.08 rows=33 width=12) + Output: pt1.v1, pt1.v2, pt2_1.v3 + Hash Cond: (pt1.v1 = pt2_1.v3) + -> Append (cost=0.00..4.50 rows=33 width=8) + Partition Selectors: $0, $1 + -> Seq Scan on public.pt1_p1 pt1_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt1_1.v1, pt1_1.v2 + -> Seq Scan on public.pt1_p2 pt1_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_2.v1, pt1_2.v2 + -> Seq Scan on public.pt1_p3 pt1_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_3.v1, pt1_3.v2 + -> Seq Scan on public.pt1_p4 pt1_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt1_4.v1, pt1_4.v2 + -> Hash (cost=8.71..8.71 rows=33 width=4) + Output: pt2_1.v3 + -> Partition Selector (selector id: $0) (cost=2.42..8.71 rows=33 width=4) + Output: pt2_1.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.42..8.71 rows=33 width=4) + Output: pt2_1.v3 + Hash Key: pt2_1.v3 + -> Hash Join (cost=2.42..8.04 rows=33 width=4) + Output: pt2_1.v3 + Hash Cond: (pt2_1.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..5.17 rows=33 width=8) + Output: pt2_1.v4, pt2_1.v3 + Hash Key: pt2_1.v4 + -> Append (cost=0.00..4.50 rows=33 width=8) + -> Seq Scan on public.pt2_p1 pt2_6 (cost=0.00..1.06 rows=6 width=8) + Output: pt2_6.v4, pt2_6.v3 + -> Seq Scan on public.pt2_p2 pt2_7 (cost=0.00..1.07 rows=7 width=8) + Output: pt2_7.v4, pt2_7.v3 + -> Seq Scan on public.pt2_p3 pt2_8 (cost=0.00..1.07 rows=7 width=8) + Output: pt2_8.v4, pt2_8.v3 + -> Seq Scan on public.pt2_p4 pt2_9 (cost=0.00..1.14 rows=14 width=8) + Output: pt2_9.v4, pt2_9.v3 + -> Hash (cost=2.00..2.00 rows=33 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..2.00 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + -> Hash (cost=4.50..4.50 rows=33 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $1) (cost=0.00..4.50 rows=33 width=4) + Output: pt2.v3 + -> Append (cost=0.00..4.50 rows=33 width=4) + -> Seq Scan on public.pt2_p1 pt2_2 (cost=0.00..1.06 rows=6 width=4) + Output: pt2_2.v3 + -> Seq Scan on public.pt2_p2 pt2_3 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_3.v3 + -> Seq Scan on public.pt2_p3 pt2_4 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_4.v3 + -> Seq Scan on public.pt2_p4 pt2_5 (cost=0.00..1.14 rows=14 width=4) + Output: pt2_5.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(62 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2,t3 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=9.12..15.42 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=9.12..14.08 rows=33 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Append (cost=0.00..4.50 rows=33 width=8) + Partition Selectors: $0 + -> Seq Scan on public.pt1_p1 pt1_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt1_1.v1, pt1_1.v2 + -> Seq Scan on public.pt1_p2 pt1_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_2.v1, pt1_2.v2 + -> Seq Scan on public.pt1_p3 pt1_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_3.v1, pt1_3.v2 + -> Seq Scan on public.pt1_p4 pt1_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt1_4.v1, pt1_4.v2 + -> Hash (cost=8.71..8.71 rows=33 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=2.42..8.71 rows=33 width=4) + Output: pt2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.42..8.71 rows=33 width=4) + Output: pt2.v3 + Hash Key: pt2.v3 + -> Hash Join (cost=2.42..8.04 rows=33 width=4) + Output: pt2.v3 + Hash Cond: (pt2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..5.17 rows=33 width=8) + Output: pt2.v4, pt2.v3 + Hash Key: pt2.v4 + -> Append (cost=0.00..4.50 rows=33 width=8) + -> Seq Scan on public.pt2_p1 pt2_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt2_1.v4, pt2_1.v3 + -> Seq Scan on public.pt2_p2 pt2_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt2_2.v4, pt2_2.v3 + -> Seq Scan on public.pt2_p3 pt2_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt2_3.v4, pt2_3.v3 + -> Seq Scan on public.pt2_p4 pt2_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt2_4.v4, pt2_4.v3 + -> Hash (cost=2.00..2.00 rows=33 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..2.00 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(46 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=15.46..15.56 rows=7 width=8) + Output: pt1.v1, pt1.v2 + -> Result (cost=15.46..15.47 rows=2 width=8) + Output: pt1.v1, pt1.v2 + -> Unique (cost=15.46..15.47 rows=2 width=8) + Output: pt1.v1, pt1.v2, (RowIdExpr) + Group Key: (RowIdExpr) + -> Sort (cost=15.46..15.47 rows=2 width=8) + Output: pt1.v1, pt1.v2, (RowIdExpr) + Sort Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=10.76..15.45 rows=2 width=8) + Output: pt1.v1, pt1.v2, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=10.76..15.40 rows=2 width=8) + Output: pt1.v1, pt1.v2, (RowIdExpr) + Hash Cond: (pt2.v3 = pt1.v1) + -> Append (cost=0.00..4.50 rows=33 width=4) + Partition Selectors: $0 + -> Seq Scan on public.pt2_p1 pt2_2 (cost=0.00..1.06 rows=6 width=4) + Output: pt2_2.v3 + -> Seq Scan on public.pt2_p2 pt2_3 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_3.v3 + -> Seq Scan on public.pt2_p3 pt2_4 (cost=0.00..1.07 rows=7 width=4) + Output: pt2_4.v3 + -> Seq Scan on public.pt2_p4 pt2_5 (cost=0.00..1.14 rows=14 width=4) + Output: pt2_5.v3 + -> Hash (cost=10.73..10.73 rows=2 width=12) + Output: pt1.v1, pt1.v2, pt2_1.v3, (RowIdExpr) + -> Partition Selector (selector id: $0) (cost=6.12..10.73 rows=2 width=12) + Output: pt1.v1, pt1.v2, pt2_1.v3, (RowIdExpr) + -> Hash Semi Join (cost=6.12..10.73 rows=2 width=12) + Output: pt1.v1, pt1.v2, pt2_1.v3, RowIdExpr + Hash Cond: (pt1.v1 = pt2_1.v3) + -> Append (cost=0.00..4.50 rows=33 width=8) + Partition Selectors: $1 + -> Seq Scan on public.pt1_p1 pt1_1 (cost=0.00..1.06 rows=6 width=8) + Output: pt1_1.v1, pt1_1.v2 + -> Seq Scan on public.pt1_p2 pt1_2 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_2.v1, pt1_2.v2 + -> Seq Scan on public.pt1_p3 pt1_3 (cost=0.00..1.07 rows=7 width=8) + Output: pt1_3.v1, pt1_3.v2 + -> Seq Scan on public.pt1_p4 pt1_4 (cost=0.00..1.14 rows=14 width=8) + Output: pt1_4.v1, pt1_4.v2 + -> Hash (cost=6.11..6.11 rows=1 width=4) + Output: pt2_1.v3 + -> Partition Selector (selector id: $1) (cost=1.63..6.11 rows=1 width=4) + Output: pt2_1.v3 + -> Hash Join (cost=1.63..6.11 rows=1 width=4) + Output: pt2_1.v3 + Hash Cond: (pt2_1.v4 = t3.v6) + -> Append (cost=0.00..4.45 rows=6 width=8) + -> Seq Scan on public.pt2_p1 pt2_6 (cost=0.00..1.08 rows=3 width=8) + Output: pt2_6.v4, pt2_6.v3 + Filter: (pt2_6.v4 < 10) + -> Seq Scan on public.pt2_p2 pt2_7 (cost=0.00..1.08 rows=1 width=8) + Output: pt2_7.v4, pt2_7.v3 + Filter: (pt2_7.v4 < 10) + -> Seq Scan on public.pt2_p3 pt2_8 (cost=0.00..1.08 rows=1 width=8) + Output: pt2_8.v4, pt2_8.v3 + Filter: (pt2_8.v4 < 10) + -> Seq Scan on public.pt2_p4 pt2_9 (cost=0.00..1.17 rows=1 width=8) + Output: pt2_9.v4, pt2_9.v3 + Filter: (pt2_9.v4 < 10) + -> Hash (cost=1.52..1.52 rows=8 width=4) + Output: t3.v6 + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..1.52 rows=8 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.42 rows=3 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(72 rows) + +select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 9 | 9 + 8 | 8 + 6 | 6 + 7 | 7 + 1 | 1 + 5 | 5 +(9 rows) + +select * from pt1 where v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 5 | 5 + 6 | 6 + 9 | 9 + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 1 | 1 +(9 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=10000000002.08..10000000192.76 rows=5000 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Nested Loop (cost=10000000002.08..10000000126.10 rows=1667 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: (SubPlan 1) + -> Hash Semi Join (cost=2.08..3.87 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=8) + Output: t2.v3, t2.v4 + SubPlan 1 + -> Nested Loop (cost=10000000000.00..10000000132.58 rows=10000 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t2_2.v3 + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.33 rows=33 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..4.67 rows=100 width=4) + Output: t3.v6 + -> Result (cost=0.00..4.17 rows=100 width=4) + Output: t3.v6 + Filter: (t2.v4 = t3.v6) + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t3.v6 + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(42 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from t3,cte1 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=10000000002.08..10000000192.76 rows=5000 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Nested Loop (cost=10000000002.08..10000000126.10 rows=1667 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: (SubPlan 1) + -> Hash Semi Join (cost=2.08..3.87 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=8) + Output: t2.v3, t2.v4 + SubPlan 1 + -> Nested Loop (cost=10000000000.00..10000000132.58 rows=10000 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t2_2.v3 + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.33 rows=33 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..4.67 rows=100 width=4) + Output: t3.v6 + -> Result (cost=0.00..4.17 rows=100 width=4) + Output: t3.v6 + Filter: (t2.v4 = t3.v6) + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t3.v6 + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(42 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=10000000000.00..10000000190.22 rows=5000 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Nested Loop (cost=10000000000.00..10000000123.56 rows=1667 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: (SubPlan 1) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Materialize (cost=0.00..3.17 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=8) + Output: t2.v3, t2.v4 + SubPlan 1 + -> Result (cost=10000000000.00..10000000132.59 rows=10000 width=4) + Output: t2_1.v3 + One-Time Filter: (t2.v4 < 10) + -> Nested Loop (cost=10000000000.00..10000000132.59 rows=10000 width=4) + Output: t2_1.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t2_1.v3 + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Materialize (cost=0.00..4.67 rows=100 width=4) + Output: t3.v6 + -> Result (cost=0.00..4.17 rows=100 width=4) + Output: t3.v6 + Filter: (t2.v4 = t3.v6) + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t3.v6 + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(38 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=10000000002.08..10000000192.76 rows=5000 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Nested Loop (cost=10000000002.08..10000000126.10 rows=1667 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: (SubPlan 1) + -> Hash Semi Join (cost=2.08..3.87 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=8) + Output: t2.v3, t2.v4 + SubPlan 1 + -> Result (cost=10000000000.00..10000000132.59 rows=10000 width=4) + Output: t2_2.v3 + One-Time Filter: (t2.v4 < 10) + -> Nested Loop (cost=10000000000.00..10000000132.59 rows=10000 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t2_2.v3 + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.33 rows=33 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..4.67 rows=100 width=4) + Output: t3.v6 + -> Result (cost=0.00..4.17 rows=100 width=4) + Output: t3.v6 + Filter: (t2.v4 = t3.v6) + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t3.v6 + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(45 rows) + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + sum +------- + 45450 +(1 row) + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + sum +------- + 45450 +(1 row) + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=10000000002.08..10000000192.76 rows=5000 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Nested Loop (cost=10000000002.08..10000000126.10 rows=1667 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: (SubPlan 1) + -> Hash Semi Join (cost=2.08..3.87 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.67..1.67 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..2.67 rows=100 width=8) + Output: t2.v3, t2.v4 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=8) + Output: t2.v3, t2.v4 + SubPlan 1 + -> Result (cost=10000000000.00..10000000132.59 rows=10000 width=4) + Output: t2_2.v3 + One-Time Filter: (t2.v4 < 10) + -> Nested Loop (cost=10000000000.00..10000000132.59 rows=10000 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t2_2.v3 + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t2_2.v3 + -> Seq Scan on public.t2 t2_2 (cost=0.00..1.33 rows=33 width=4) + Output: t2_2.v3 + -> Materialize (cost=0.00..4.67 rows=100 width=4) + Output: t3.v6 + -> Result (cost=0.00..4.17 rows=100 width=4) + Output: t3.v6 + Filter: (t2.v4 = t3.v6) + -> Materialize (cost=0.00..3.17 rows=100 width=4) + Output: t3.v6 + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..2.67 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'off', optimizer_cte_inlining_bound = '2' + Optimizer: Postgres query optimizer +(45 rows) + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + sum +------- + 45450 +(1 row) + +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; +-- dedup the subquery with semi join(still be a scalar in the pre-process) +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + QUERY PLAN +---------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=5.71..9.29 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=5.71..7.96 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Hash Semi Join (cost=3.96..5.75 rows=33 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t3.v5 + Hash Cond: (t1.v1 = t3.v5) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=3.54..3.54 rows=33 width=8) + Output: t2_1.v3, t3.v5 + -> Hash Semi Join (cost=1.75..3.54 rows=33 width=8) + Output: t2_1.v3, t3.v5 + Hash Cond: (t2_1.v3 = t3.v5) + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3, t2_1.v4 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t3.v5 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v5 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(27 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + QUERY PLAN +---------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=3.96..7.08 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=3.96..5.75 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t3.v5) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=3.54..3.54 rows=33 width=8) + Output: t2.v3, t3.v5 + -> Hash Semi Join (cost=1.75..3.54 rows=33 width=8) + Output: t2.v3, t3.v5 + Hash Cond: (t2.v3 = t3.v5) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3, t2.v4 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t3.v5 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v5 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(20 rows) + +select count(*) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + count +------- + 100 +(1 row) + +select count(*) from t1 where v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + count +------- + 100 +(1 row) + +-- inner join different key +explain verbose select v1 from t1 where v1 in (select v3 from t2) and v1 in (select v5 from t2,t3 where v5 = v3); + QUERY PLAN +---------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=5.71..9.29 rows=100 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=5.71..7.96 rows=33 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2.v3) + -> Hash Semi Join (cost=3.96..5.75 rows=33 width=12) + Output: t1.v1, t2_1.v3, t3.v5 + Hash Cond: (t1.v1 = t3.v5) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=3.54..3.54 rows=33 width=8) + Output: t2_1.v3, t3.v5 + -> Hash Join (cost=1.75..3.54 rows=33 width=8) + Output: t2_1.v3, t3.v5 + Hash Cond: (t2_1.v3 = t3.v5) + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3, t2_1.v4 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t3.v5 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v5 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(27 rows) + +explain verbose select v1 from t1 where v1 in (select v5 from t2,t3 where v5 = v3); + QUERY PLAN +---------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=3.96..7.08 rows=100 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=3.96..5.75 rows=33 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t3.v5) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=3.54..3.54 rows=33 width=8) + Output: t2.v3, t3.v5 + -> Hash Join (cost=1.75..3.54 rows=33 width=8) + Output: t2.v3, t3.v5 + Hash Cond: (t2.v3 = t3.v5) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3, t2.v4 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t3.v5 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=4) + Output: t3.v5 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(20 rows) + +explain verbose select v1 from t1 where v1 in (select v3 from t2) and v1 in (select v6 from t2,t3 where v5 = v3); -- no dedup, because v6 is not the join key + QUERY PLAN +----------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=6.38..9.96 rows=100 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=6.38..8.62 rows=33 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2.v3) + -> Hash Semi Join (cost=4.63..6.42 rows=33 width=8) + Output: t1.v1, t3.v6 + Hash Cond: (t1.v1 = t3.v6) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=4) + Output: t1.v1, t1.v2 + -> Hash (cost=4.21..4.21 rows=33 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.75..4.21 rows=33 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Hash Join (cost=1.75..3.54 rows=33 width=4) + Output: t3.v6 + Hash Cond: (t2_1.v3 = t3.v5) + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3, t2_1.v4 + -> Hash (cost=1.33..1.33 rows=33 width=8) + Output: t3.v5, t3.v6 + -> Seq Scan on public.t3 (cost=0.00..1.33 rows=33 width=8) + Output: t3.v5, t3.v6 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(30 rows) + +select count(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v5 from t2,t3 where v5 = v3); + count +------- + 100 +(1 row) + +select count(v1) from t1 where v1 in (select v5 from t2,t3 where v5 = v3); + count +------- + 100 +(1 row) + +-- can't dedup +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v4 from t2 where v3 < 10); -- different outpt + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=4.54..4.67 rows=8 width=8) + Output: t1.v1, t1.v2 + -> Result (cost=4.54..4.56 rows=3 width=8) + Output: t1.v1, t1.v2 + -> Unique (cost=4.54..4.56 rows=3 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Group Key: (RowIdExpr) + -> Sort (cost=4.54..4.55 rows=3 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Sort Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.99..4.53 rows=3 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=2.99..4.47 rows=3 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Cond: (t2.v3 = t1.v1) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3, t2.v4 + -> Hash (cost=2.95..2.95 rows=3 width=12) + Output: t1.v1, t1.v2, t2_1.v4, (RowIdExpr) + -> Hash Semi Join (cost=1.50..2.95 rows=3 width=12) + Output: t1.v1, t1.v2, t2_1.v4, RowIdExpr + Hash Cond: (t1.v1 = t2_1.v4) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.47..1.47 rows=3 width=4) + Output: t2_1.v4 + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..1.47 rows=3 width=4) + Output: t2_1.v4 + Hash Key: t2_1.v4 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v4 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(35 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 where v3 < 10); -- different scalar ident + QUERY PLAN +---------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=4.43..4.49 rows=3 width=8) + Output: t1.v1, t1.v2 + -> HashAggregate (cost=4.43..4.44 rows=1 width=8) + Output: t1.v1, t1.v2 + Group Key: (RowIdExpr) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=2.98..4.43 rows=1 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Key: (RowIdExpr) + -> Hash Join (cost=2.98..4.41 rows=1 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Cond: (t2.v3 = t1.v1) + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3, t2.v4 + -> Hash (cost=2.96..2.96 rows=1 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=1.45..2.96 rows=1 width=8) + Output: t1.v1, t1.v2, (RowIdExpr) + Hash Key: t1.v1 + -> Hash Semi Join (cost=1.45..2.94 rows=1 width=8) + Output: t1.v1, t1.v2, RowIdExpr + Hash Cond: (t1.v2 = t2_1.v3) + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..1.47 rows=3 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Seq Scan on public.t1 (cost=0.00..1.42 rows=3 width=8) + Output: t1.v1, t1.v2 + Filter: (t1.v2 < 10) + -> Hash (cost=1.42..1.42 rows=3 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.42 rows=3 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(34 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 group by v3); -- group by + QUERY PLAN +----------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=3.50..7.75 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=3.50..6.42 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v2 = t2_1.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.75..4.21 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Hash Semi Join (cost=1.75..3.54 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(23 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 order by v3); -- order by/limit, actully this case can be the subset + QUERY PLAN +----------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=3.50..7.75 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=3.50..6.42 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v2 = t2_1.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=1.75..4.21 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Hash Semi Join (cost=1.75..3.54 rows=33 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..1.33 rows=33 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..1.33 rows=33 width=4) + Output: t2.v3 + -> Hash (cost=1.33..1.33 rows=33 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..1.33 rows=33 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'off' + Optimizer: Postgres query optimizer +(23 rows) + +reset optimizer_trace_fallback; +-- start_ignore +drop table if exists t1; +drop table if exists t2; +drop table if exists t3; +drop table if exists pt1; +drop table if exists pt2; +-- end_ignore diff --git a/src/test/regress/expected/dedupset_optimizer.out b/src/test/regress/expected/dedupset_optimizer.out new file mode 100644 index 00000000000..f9369e5a87f --- /dev/null +++ b/src/test/regress/expected/dedupset_optimizer.out @@ -0,0 +1,1614 @@ +SET optimizer_trace_fallback = on; +-- start_ignore +drop table if exists t1; +drop table if exists t2; +drop table if exists t3; +drop table if exists pt1; +drop table if exists pt2; +-- end_ignore +create table t1(v1 int, v2 int); +create table t2(v3 int, v4 int); +create table t3(v5 int, v6 int); +insert into t1 values(generate_series(1, 100), generate_series(1, 100)); +insert into t2 values(generate_series(1, 100), generate_series(1, 100)); +insert into t3 values(generate_series(1, 100), generate_series(1, 100)); +CREATE TABLE pt1 ( + v1 INT, + v2 INT +) PARTITION BY RANGE (v1); +CREATE TABLE pt1_p1 PARTITION OF pt1 FOR VALUES FROM (0) TO (20); +CREATE TABLE pt1_p2 PARTITION OF pt1 FOR VALUES FROM (20) TO (40); +CREATE TABLE pt1_p3 PARTITION OF pt1 FOR VALUES FROM (40) TO (60); +CREATE TABLE pt1_p4 PARTITION OF pt1 FOR VALUES FROM (60) TO (400); +CREATE TABLE pt2 ( + v3 INT, + v4 INT +) PARTITION BY RANGE (v3); +CREATE TABLE pt2_p1 PARTITION OF pt2 FOR VALUES FROM (0) TO (20); +CREATE TABLE pt2_p2 PARTITION OF pt2 FOR VALUES FROM (20) TO (40); +CREATE TABLE pt2_p3 PARTITION OF pt2 FOR VALUES FROM (40) TO (60); +CREATE TABLE pt2_p4 PARTITION OF pt2 FOR VALUES FROM (60) TO (400); +insert into pt1 values(generate_series(1, 100), generate_series(1, 100)); +insert into pt2 values(generate_series(1, 100), generate_series(1, 100)); +analyze t1; +analyze t2; +analyze t3; +analyze pt1; +analyze pt2; +-- dedup the subquery with projection +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(14 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10) and v1 in (select v3 from t2); -- change the order + QUERY PLAN +------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(14 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(14 rows) + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 1 | 1 + 5 | 5 + 6 | 6 + 9 | 9 +(9 rows) + +select * from t1 where v1 in (select v3 from t2 where v3 < 10); + v1 | v2 +----+---- + 1 | 1 + 5 | 5 + 6 | 6 + 9 | 9 + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 +(9 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2 where v3 < 10); + QUERY PLAN +-------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.00 rows=10 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..862.00 rows=4 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=4 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=4 width=4) + Output: pt2.v3 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=4 width=4) + Output: pt2.v3 + Number of partitions to scan: 1 (out of 4) + Filter: (pt2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(18 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2 where v3 < 10); + QUERY PLAN +-------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.00 rows=10 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..862.00 rows=4 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=4 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=4 width=4) + Output: pt2.v3 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=4 width=4) + Output: pt2.v3 + Number of partitions to scan: 1 (out of 4) + Filter: (pt2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(18 rows) + +select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2 where v3 < 10); + v1 | v2 +----+---- + 1 | 1 + 5 | 5 + 6 | 6 + 9 | 9 + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 +(9 rows) + +select * from pt1 where v1 in (select v3 from pt2 where v3 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 5 | 5 + 6 | 6 + 9 | 9 + 1 | 1 +(9 rows) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1324954.52 rows=1 width=8) + Output: (count(*)) + -> Sequence (cost=0.00..1324954.52 rows=1 width=8) + Output: (count(*)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=1) + Output: share0_ref1.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Redistribute Motion 1:3 (slice2) (cost=0.00..1324523.52 rows=1 width=8) + Output: (count(*)) + -> Finalize Aggregate (cost=0.00..1324523.52 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..1324523.52 rows=1 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=0.00..1324523.52 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=0.00..1324523.52 rows=304 width=1) + Hash Cond: (t1.v1 = share0_ref2.v3) + -> Nested Loop (cost=0.00..1324091.93 rows=3334 width=4) + Output: t1.v1 + Join Filter: true + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Materialize (cost=0.00..431.00 rows=100 width=1) + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=100 width=1) + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=1) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: share0_ref2.v3 + -> Result (cost=0.00..431.00 rows=4 width=4) + Output: share0_ref2.v3 + Filter: (share0_ref2.v3 < 10) + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=34 width=4) + Output: share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(35 rows) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1 where v3 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=0.00..1324523.52 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1324523.52 rows=1 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=0.00..1324523.51 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=0.00..1324523.51 rows=304 width=1) + Hash Cond: (t1.v1 = t2_1.v3) + -> Nested Loop (cost=0.00..1324091.93 rows=3334 width=4) + Output: t1.v1 + Join Filter: true + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Materialize (cost=0.00..431.00 rows=100 width=1) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=100 width=1) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=1) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=4 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(23 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + count +------- + 900 +(1 row) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1 where v3 < 10); + count +------- + 900 +(1 row) + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=0.00..1324523.52 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1324523.52 rows=1 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=0.00..1324523.52 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=0.00..1324523.52 rows=304 width=1) + Hash Cond: (t1.v1 = t2_1.v3) + -> Nested Loop (cost=0.00..1324091.93 rows=3334 width=4) + Output: t1.v1 + Join Filter: true + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Materialize (cost=0.00..431.00 rows=100 width=1) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=100 width=1) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=1) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=4 width=4) + Output: t2_1.v3 + Filter: (t2_1.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on', optimizer_cte_inlining_bound = '2' + Optimizer: GPORCA +(23 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); + count +------- + 900 +(1 row) + +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------- + Finalize Aggregate (cost=0.00..862.01 rows=1 width=8) + Output: sum(t1.v1) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=1 width=8) + Output: (PARTIAL sum(t1.v1)) + -> Partial Aggregate (cost=0.00..862.01 rows=1 width=8) + Output: PARTIAL sum(t1.v1) + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(18 rows) + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------- + Finalize Aggregate (cost=0.00..862.01 rows=1 width=8) + Output: sum(t1.v1) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=1 width=8) + Output: (PARTIAL sum(t1.v1)) + -> Partial Aggregate (cost=0.00..862.01 rows=1 width=8) + Output: PARTIAL sum(t1.v1) + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(18 rows) + +select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); + sum +----- + 45 +(1 row) + +select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10); + sum +----- + 45 +(1 row) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) order by v1; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: t1.v1, t1.v2 + Merge Key: t1.v1 + -> Sort (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v1 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(18 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10) order by v1; + QUERY PLAN +----------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: t1.v1, t1.v2 + Merge Key: t1.v1 + -> Sort (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v1 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(18 rows) + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) order by v1; + v1 | v2 +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 +(9 rows) + +select * from t1 where v1 in (select v3 from t2 where v3 < 10) order by v1; + v1 | v2 +----+---- + 1 | 1 + 2 | 2 + 3 | 3 + 4 | 4 + 5 | 5 + 6 | 6 + 7 | 7 + 8 | 8 + 9 | 9 +(9 rows) + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) group by v2; + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: (sum(t1.v1)) + -> GroupAggregate (cost=0.00..862.01 rows=4 width=8) + Output: sum(t1.v1) + Group Key: t1.v2 + -> Sort (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(23 rows) + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10) group by v2; + QUERY PLAN +------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.01 rows=10 width=8) + Output: (sum(t1.v1)) + -> GroupAggregate (cost=0.00..862.01 rows=4 width=8) + Output: sum(t1.v1) + Group Key: t1.v2 + -> Sort (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Sort Key: t1.v2 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(23 rows) + +select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) group by v2; + sum +----- + 1 + 5 + 6 + 9 + 2 + 3 + 4 + 7 + 8 +(9 rows) + +select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10) group by v2; + sum +----- + 2 + 3 + 4 + 7 + 8 + 1 + 5 + 6 + 9 +(9 rows) + +-- dedup the same subqueryany +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2); + QUERY PLAN +-------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.02 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(13 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2); + QUERY PLAN +-------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.02 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(13 rows) + +select count(*) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2); + count +------- + 100 +(1 row) + +select count(*) from t1 where v1 in (select v3 from t2); + count +------- + 100 +(1 row) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.02 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=34 width=4) + Output: pt2.v3 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=34 width=4) + Output: pt2.v3 + Number of partitions to scan: 4 (out of 4) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(17 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2); + QUERY PLAN +--------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..862.02 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..431.00 rows=34 width=4) + Output: pt2.v3 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=34 width=4) + Output: pt2.v3 + Number of partitions to scan: 4 (out of 4) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(17 rows) + +select count(*) from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2); + count +------- + 100 +(1 row) + +select count(*) from pt1 where v1 in (select v3 from pt2); + count +------- + 100 +(1 row) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1324954.53 rows=1 width=8) + Output: (count(*)) + -> Sequence (cost=0.00..1324954.53 rows=1 width=8) + Output: (count(*)) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=1) + Output: share0_ref1.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Redistribute Motion 1:3 (slice2) (cost=0.00..1324523.53 rows=1 width=8) + Output: (count(*)) + -> Finalize Aggregate (cost=0.00..1324523.53 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice3; segments: 3) (cost=0.00..1324523.53 rows=1 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=0.00..1324523.53 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=0.00..1324523.53 rows=3334 width=1) + Hash Cond: (t1.v1 = share0_ref2.v3) + -> Nested Loop (cost=0.00..1324091.93 rows=3334 width=4) + Output: t1.v1 + Join Filter: true + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Materialize (cost=0.00..431.00 rows=100 width=1) + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=100 width=1) + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=1) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: share0_ref2.v3 + -> Shared Scan (share slice:id 3:0) (cost=0.00..431.00 rows=34 width=4) + Output: share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(32 rows) + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=0.00..1324523.53 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1324523.53 rows=1 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=0.00..1324523.53 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=0.00..1324523.53 rows=3334 width=1) + Hash Cond: (t1.v1 = t2_1.v3) + -> Nested Loop (cost=0.00..1324091.93 rows=3334 width=4) + Output: t1.v1 + Join Filter: true + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Materialize (cost=0.00..431.00 rows=100 width=1) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=100 width=1) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=1) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(22 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + count +------- + 10000 +(1 row) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1); + count +------- + 10000 +(1 row) + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Finalize Aggregate (cost=0.00..1324523.53 rows=1 width=8) + Output: count(*) + -> Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1324523.53 rows=1 width=8) + Output: (PARTIAL count(*)) + -> Partial Aggregate (cost=0.00..1324523.53 rows=1 width=8) + Output: PARTIAL count(*) + -> Hash Semi Join (cost=0.00..1324523.53 rows=3334 width=1) + Hash Cond: (t1.v1 = t2_1.v3) + -> Nested Loop (cost=0.00..1324091.93 rows=3334 width=4) + Output: t1.v1 + Join Filter: true + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Materialize (cost=0.00..431.00 rows=100 width=1) + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=100 width=1) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=1) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on', optimizer_cte_inlining_bound = '2' + Optimizer: GPORCA +(22 rows) + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); + count +------- + 10000 +(1 row) + +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; +-- dedup the subquery with inner join +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: t2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Key: t2.v3 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Cond: (t2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + Hash Key: t2.v4 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(29 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t3,t2 where v4=v6); -- change the join order,different index in inner join + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: t2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Key: t2.v3 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Cond: (t2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + Hash Key: t2.v4 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(29 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2,t3 where v4=v6); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: t2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Key: t2.v3 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Cond: (t2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + Hash Key: t2.v4 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(29 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.01 rows=10 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=862.01..862.01 rows=4 width=4) + Output: t2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.01 rows=4 width=4) + Output: t2.v3 + Hash Key: t2.v3 + -> Hash Join (cost=0.00..862.01 rows=4 width=4) + Output: t2.v3 + Hash Cond: (t2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=8) + Output: t2.v3, t2.v4 + Hash Key: t2.v4 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=8) + Output: t2.v3, t2.v4 + Filter: (t2.v4 < 10) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(31 rows) + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 9 | 9 + 8 | 8 + 6 | 6 + 7 | 7 + 1 | 1 + 5 | 5 +(9 rows) + +select * from t1 where v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 5 | 5 + 6 | 6 + 9 | 9 + 1 | 1 +(9 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + Hash Key: pt2.v3 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + Hash Cond: (pt2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: pt2.v3, pt2.v4 + Hash Key: pt2.v4 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=34 width=8) + Output: pt2.v3, pt2.v4 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(33 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from t3,pt2 where v4=v6); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + Hash Key: pt2.v3 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + Hash Cond: (pt2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: pt2.v3, pt2.v4 + Hash Key: pt2.v4 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=34 width=8) + Output: pt2.v3, pt2.v4 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(33 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2,t3 where v4=v6); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=34 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + Hash Key: pt2.v3 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: pt2.v3 + Hash Cond: (pt2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: pt2.v3, pt2.v4 + Hash Key: pt2.v4 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=34 width=8) + Output: pt2.v3, pt2.v4 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(33 rows) + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + QUERY PLAN +------------------------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.01 rows=10 width=8) + Output: pt1.v1, pt1.v2 + -> Hash Semi Join (cost=0.00..1293.01 rows=4 width=8) + Output: pt1.v1, pt1.v2 + Hash Cond: (pt1.v1 = pt2.v3) + -> Dynamic Seq Scan on public.pt1 (cost=0.00..431.00 rows=4 width=8) + Output: pt1.v1, pt1.v2 + Number of partitions to scan: 4 (out of 4) + -> Hash (cost=862.01..862.01 rows=4 width=4) + Output: pt2.v3 + -> Partition Selector (selector id: $0) (cost=0.00..862.01 rows=4 width=4) + Output: pt2.v3 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.01 rows=4 width=4) + Output: pt2.v3 + Hash Key: pt2.v3 + -> Hash Join (cost=0.00..862.01 rows=4 width=4) + Output: pt2.v3 + Hash Cond: (pt2.v4 = t3.v6) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=4 width=8) + Output: pt2.v3, pt2.v4 + Hash Key: pt2.v4 + -> Dynamic Seq Scan on public.pt2 (cost=0.00..431.00 rows=4 width=8) + Output: pt2.v3, pt2.v4 + Number of partitions to scan: 4 (out of 4) + Filter: (pt2.v4 < 10) + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(35 rows) + +select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 2 | 2 + 3 | 3 + 4 | 4 + 9 | 9 + 8 | 8 + 6 | 6 + 7 | 7 + 1 | 1 + 5 | 5 +(9 rows) + +select * from pt1 where v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + v1 | v2 +----+---- + 5 | 5 + 6 | 6 + 9 | 9 + 2 | 2 + 3 | 3 + 4 | 4 + 7 | 7 + 8 | 8 + 1 | 1 +(9 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2648910.08 rows=5625 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + -> Sequence (cost=0.00..2648909.74 rows=1875 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=1) + Output: share0_ref1.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash Semi Join (cost=0.00..2648478.71 rows=1875 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Hash Cond: ((t2_1.v4 = t3.v6) AND (t1.v1 = share0_ref2.v3)) + -> Nested Loop (cost=0.00..1324306.71 rows=3334 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Join Filter: true + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.02 rows=100 width=8) + Output: t2_1.v3, t2_1.v4 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=8) + Output: t2_1.v3, t2_1.v4 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1324169.43..1324169.43 rows=3334 width=8) + Output: share0_ref2.v3, t3.v6 + -> Nested Loop (cost=0.00..1324169.43 rows=3334 width=8) + Output: share0_ref2.v3, t3.v6 + Join Filter: true + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..431.01 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=4) + Output: share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(33 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from t3,cte1 where v4=v6); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2648910.08 rows=5625 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + -> Sequence (cost=0.00..2648909.74 rows=1875 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=1) + Output: share0_ref1.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash Semi Join (cost=0.00..2648478.71 rows=1875 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Hash Cond: ((t2_1.v4 = t3.v6) AND (t1.v1 = share0_ref2.v3)) + -> Nested Loop (cost=0.00..1324306.71 rows=3334 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Join Filter: true + -> Broadcast Motion 3:3 (slice2; segments: 3) (cost=0.00..431.02 rows=100 width=8) + Output: t2_1.v3, t2_1.v4 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=8) + Output: t2_1.v3, t2_1.v4 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=1324169.43..1324169.43 rows=3334 width=8) + Output: t3.v6, share0_ref2.v3 + -> Nested Loop (cost=0.00..1324169.43 rows=3334 width=8) + Output: t3.v6, share0_ref2.v3 + Join Filter: true + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..431.01 rows=100 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v6 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=4) + Output: share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(33 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2648354.45 rows=512 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Hash Semi Join (cost=0.00..2648354.42 rows=171 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Hash Cond: ((t2.v4 = t3.v6) AND (t1.v1 = t2_1.v3)) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1324306.84 rows=304 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Hash Key: t1.v1 + -> Result (cost=0.00..1324306.82 rows=304 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Filter: (t2.v4 < 10) + -> Nested Loop (cost=0.00..1324306.71 rows=3334 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: true + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..431.02 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + -> Hash (cost=1324047.35..1324047.35 rows=304 width=8) + Output: t2_1.v3, t3.v6 + -> Nested Loop (cost=0.00..1324047.35 rows=304 width=8) + Output: t2_1.v3, t3.v6 + Join Filter: true + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=10 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(34 rows) + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2648784.74 rows=512 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + -> Sequence (cost=0.00..2648784.71 rows=171 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=1) + Output: share0_ref1.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash Semi Join (cost=0.00..2648353.70 rows=171 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Hash Cond: ((t2_1.v4 = t3.v6) AND (t1.v1 = share0_ref2.v3)) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1324306.84 rows=304 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Hash Key: t1.v1 + -> Result (cost=0.00..1324306.82 rows=304 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Filter: (t2_1.v4 < 10) + -> Nested Loop (cost=0.00..1324306.71 rows=3334 width=16) + Output: t1.v1, t1.v2, t2_1.v3, t2_1.v4 + Join Filter: true + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..431.02 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=8) + Output: t2_1.v3, t2_1.v4 + -> Hash (cost=1324046.63..1324046.63 rows=304 width=8) + Output: share0_ref2.v3, t3.v6 + -> Nested Loop (cost=0.00..1324046.63 rows=304 width=8) + Output: share0_ref2.v3, t3.v6 + Join Filter: true + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=10 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + -> Shared Scan (share slice:id 1:0) (cost=0.00..431.00 rows=34 width=4) + Output: share0_ref2.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(40 rows) + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + sum +------- + 45450 +(1 row) + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + sum +------- + 45450 +(1 row) + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + QUERY PLAN +----------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..2648354.45 rows=512 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + -> Hash Semi Join (cost=0.00..2648354.42 rows=171 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Hash Cond: ((t2.v4 = t3.v6) AND (t1.v1 = t2_1.v3)) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..1324306.84 rows=304 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Hash Key: t1.v1 + -> Result (cost=0.00..1324306.82 rows=304 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Filter: (t2.v4 < 10) + -> Nested Loop (cost=0.00..1324306.71 rows=3334 width=16) + Output: t1.v1, t1.v2, t2.v3, t2.v4 + Join Filter: true + -> Broadcast Motion 3:3 (slice3; segments: 3) (cost=0.00..431.02 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=8) + Output: t2.v3, t2.v4 + -> Hash (cost=1324047.35..1324047.35 rows=304 width=8) + Output: t2_1.v3, t3.v6 + -> Nested Loop (cost=0.00..1324047.35 rows=304 width=8) + Output: t2_1.v3, t3.v6 + Join Filter: true + -> Broadcast Motion 3:3 (slice4; segments: 3) (cost=0.00..431.00 rows=10 width=4) + Output: t3.v6 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=4 width=4) + Output: t3.v6 + Filter: (t3.v6 < 10) + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on', optimizer_cte_inlining_bound = '2' + Optimizer: GPORCA +(34 rows) + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + sum +------- + 45450 +(1 row) + +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; +-- dedup the subquery with semi join(still be a scalar in the pre-process) +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + QUERY PLAN +------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: t2.v3 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Cond: (t2.v3 = t3.v5) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + Filter: (NOT (t2.v3 IS NULL)) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: (1), t3.v5 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: 1, t3.v5 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(21 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + QUERY PLAN +------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: t2.v3 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=4) + Output: t2.v3 + Hash Cond: (t2.v3 = t3.v5) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + Filter: (NOT (t2.v3 IS NULL)) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: (1), t3.v5 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: 1, t3.v5 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(21 rows) + +select count(*) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + count +------- + 100 +(1 row) + +select count(*) from t1 where v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + count +------- + 100 +(1 row) + +-- inner join different key +explain verbose select v1 from t1 where v1 in (select v3 from t2) and v1 in (select v5 from t2,t3 where v5 = v3); + QUERY PLAN +------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t3.v5) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Hash (cost=862.01..862.01 rows=34 width=4) + Output: t3.v5 + -> Hash Join (cost=0.00..862.01 rows=34 width=4) + Output: t3.v5 + Hash Cond: (t2.v3 = t3.v5) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v5 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v5 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(20 rows) + +explain verbose select v1 from t1 where v1 in (select v5 from t2,t3 where v5 = v3); + QUERY PLAN +------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t3.v5) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Hash (cost=862.01..862.01 rows=34 width=4) + Output: t3.v5 + -> Hash Join (cost=0.00..862.01 rows=34 width=4) + Output: t3.v5 + Hash Cond: (t2.v3 = t3.v5) + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t3.v5 + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=4) + Output: t3.v5 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(20 rows) + +explain verbose select v1 from t1 where v1 in (select v3 from t2) and v1 in (select v6 from t2,t3 where v5 = v3); -- no dedup, because v6 is not the join key + QUERY PLAN +------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1724.04 rows=100 width=4) + Output: t1.v1 + -> Hash Semi Join (cost=0.00..1724.04 rows=34 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=4) + Output: t1.v1 + Hash Cond: (t1.v1 = t3.v6) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=4) + Output: t1.v1 + -> Hash (cost=862.02..862.02 rows=34 width=4) + Output: t3.v6 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=4) + Output: t3.v6 + Hash Key: t3.v6 + -> Hash Join (cost=0.00..862.02 rows=34 width=4) + Output: t3.v6 + Hash Cond: (t3.v5 = t2.v3) + -> Seq Scan on public.t3 (cost=0.00..431.00 rows=34 width=8) + Output: t3.v5, t3.v6 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(30 rows) + +select count(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v5 from t2,t3 where v5 = v3); + count +------- + 100 +(1 row) + +select count(v1) from t1 where v1 in (select v5 from t2,t3 where v5 = v3); + count +------- + 100 +(1 row) + +-- can't dedup +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v4 from t2 where v3 < 10); -- different outpt + QUERY PLAN +------------------------------------------------------------------------------------------------------------ + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.02 rows=10 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.02 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v4) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v4 + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..431.00 rows=4 width=4) + Output: t2.v4 + Hash Key: t2.v4 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v4 + Filter: (t2.v3 < 10) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(24 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 where v3 < 10); -- different scalar ident + QUERY PLAN +------------------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.02 rows=10 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.02 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2_1.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v1 + -> Hash Semi Join (cost=0.00..862.01 rows=4 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v2 = t2.v3) + -> Redistribute Motion 3:3 (slice3; segments: 3) (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=4 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=4 width=4) + Output: t2.v3 + Filter: (t2.v3 < 10) + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(27 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 group by v3); -- group by + QUERY PLAN +------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.04 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v2 = t2_1.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash (cost=431.01..431.01 rows=34 width=4) + Output: t2_1.v3 + -> GroupAggregate (cost=0.00..431.01 rows=34 width=4) + Output: t2_1.v3 + Group Key: t2_1.v3 + -> Sort (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Sort Key: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(29 rows) + +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 order by v3); -- order by/limit, actully this case can be the subset + QUERY PLAN +------------------------------------------------------------------------------------------------- + Gather Motion 3:1 (slice1; segments: 3) (cost=0.00..1293.03 rows=100 width=8) + Output: t1.v1, t1.v2 + -> Hash Semi Join (cost=0.00..1293.03 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v2 = t2_1.v3) + -> Redistribute Motion 3:3 (slice2; segments: 3) (cost=0.00..862.02 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Key: t1.v2 + -> Hash Semi Join (cost=0.00..862.02 rows=34 width=8) + Output: t1.v1, t1.v2 + Hash Cond: (t1.v1 = t2.v3) + -> Seq Scan on public.t1 (cost=0.00..431.00 rows=34 width=8) + Output: t1.v1, t1.v2 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Seq Scan on public.t2 (cost=0.00..431.00 rows=34 width=4) + Output: t2.v3 + -> Hash (cost=431.00..431.00 rows=34 width=4) + Output: t2_1.v3 + -> Seq Scan on public.t2 t2_1 (cost=0.00..431.00 rows=34 width=4) + Output: t2_1.v3 + Settings: enable_parallel = 'off', optimizer = 'on' + Optimizer: GPORCA +(23 rows) + +reset optimizer_trace_fallback; +-- start_ignore +drop table if exists t1; +drop table if exists t2; +drop table if exists t3; +drop table if exists pt1; +drop table if exists pt2; +-- end_ignore diff --git a/src/test/regress/greenplum_schedule b/src/test/regress/greenplum_schedule index 09331d8d4a8..3614e349ff8 100755 --- a/src/test/regress/greenplum_schedule +++ b/src/test/regress/greenplum_schedule @@ -26,6 +26,8 @@ test: gp_dispatch_keepalives # copy form a file with different EOL test: copy_eol +test: dedupset + test: disable_autovacuum test: python_processed64bit test: enable_autovacuum diff --git a/src/test/regress/sql/cte_prune.sql b/src/test/regress/sql/cte_prune.sql index a87129d5e50..db2351b134e 100644 --- a/src/test/regress/sql/cte_prune.sql +++ b/src/test/regress/sql/cte_prune.sql @@ -222,7 +222,7 @@ explain verbose with frequent_ss_items as and d_year in (1999,1999+1,1999+2,1999+3) group by substr(i_item_desc,1,30),i_item_sk,d_date having count(*) >4) -select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items) +select t1.v1 from t1 where t1.v1 in (select item_sk from frequent_ss_items where true) and t1.v1 in (select item_sk from frequent_ss_items where item_sk > 0); -- sql 95 @@ -231,14 +231,14 @@ explain verbose with ws_wh as from tpcds_web_sales ws1,tpcds_web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -select * from t1 where t1.v1 in (select ws_order_number from ws_wh) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); +select * from t1 where t1.v1 in (select ws_order_number from ws_wh where true) and t1.v1 in (select ws_order_number from ws_wh where ws_order_number > 0); explain verbose with ws_wh as (select ws1.ws_order_number,ws1.ws_warehouse_sk wh1,ws2.ws_warehouse_sk wh2 from tpcds_web_sales ws1,tpcds_web_sales ws2 where ws1.ws_order_number = ws2.ws_order_number and ws1.ws_warehouse_sk <> ws2.ws_warehouse_sk) -select * from t1 where t1.v1 in (select wh1 from ws_wh) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); +select * from t1 where t1.v1 in (select wh1 from ws_wh where true) and t1.v1 in (select wh1 from ws_wh where ws_order_number > 0); -- start_ignore drop table tpcds_store_sales; diff --git a/src/test/regress/sql/dedupset.sql b/src/test/regress/sql/dedupset.sql new file mode 100644 index 00000000000..72ecd97ce2e --- /dev/null +++ b/src/test/regress/sql/dedupset.sql @@ -0,0 +1,174 @@ + +SET optimizer_trace_fallback = on; +-- start_ignore +drop table if exists t1; +drop table if exists t2; +drop table if exists t3; +drop table if exists pt1; +drop table if exists pt2; +-- end_ignore + +create table t1(v1 int, v2 int); +create table t2(v3 int, v4 int); +create table t3(v5 int, v6 int); + +insert into t1 values(generate_series(1, 100), generate_series(1, 100)); +insert into t2 values(generate_series(1, 100), generate_series(1, 100)); +insert into t3 values(generate_series(1, 100), generate_series(1, 100)); + +CREATE TABLE pt1 ( + v1 INT, + v2 INT +) PARTITION BY RANGE (v1); + +CREATE TABLE pt1_p1 PARTITION OF pt1 FOR VALUES FROM (0) TO (20); +CREATE TABLE pt1_p2 PARTITION OF pt1 FOR VALUES FROM (20) TO (40); +CREATE TABLE pt1_p3 PARTITION OF pt1 FOR VALUES FROM (40) TO (60); +CREATE TABLE pt1_p4 PARTITION OF pt1 FOR VALUES FROM (60) TO (400); + +CREATE TABLE pt2 ( + v3 INT, + v4 INT +) PARTITION BY RANGE (v3); + +CREATE TABLE pt2_p1 PARTITION OF pt2 FOR VALUES FROM (0) TO (20); +CREATE TABLE pt2_p2 PARTITION OF pt2 FOR VALUES FROM (20) TO (40); +CREATE TABLE pt2_p3 PARTITION OF pt2 FOR VALUES FROM (40) TO (60); +CREATE TABLE pt2_p4 PARTITION OF pt2 FOR VALUES FROM (60) TO (400); + +insert into pt1 values(generate_series(1, 100), generate_series(1, 100)); +insert into pt2 values(generate_series(1, 100), generate_series(1, 100)); + +analyze t1; +analyze t2; +analyze t3; +analyze pt1; +analyze pt2; + +-- dedup the subquery with projection +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10) and v1 in (select v3 from t2); -- change the order +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10); +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); +select * from t1 where v1 in (select v3 from t2 where v3 < 10); + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2 where v3 < 10); +explain verbose select * from pt1 where v1 in (select v3 from pt2 where v3 < 10); +select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2 where v3 < 10); +select * from pt1 where v1 in (select v3 from pt2 where v3 < 10); + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1 where v3 < 10); + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1 where v3 < 10); + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1 where v3 < 10); +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10); +select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10); +select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10); + +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) order by v1; +explain verbose select * from t1 where v1 in (select v3 from t2 where v3 < 10) order by v1; +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) order by v1; +select * from t1 where v1 in (select v3 from t2 where v3 < 10) order by v1; + +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) group by v2; +explain verbose select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10) group by v2; +select sum(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where v3 < 10) group by v2; +select sum(v1) from t1 where v1 in (select v3 from t2 where v3 < 10) group by v2; + +-- dedup the same subqueryany +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2); +explain verbose select * from t1 where v1 in (select v3 from t2); +select count(*) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2); +select count(*) from t1 where v1 in (select v3 from t2); + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2); +explain verbose select * from pt1 where v1 in (select v3 from pt2); + +select count(*) from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2); +select count(*) from pt1 where v1 in (select v3 from pt2); + +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1); + +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1); + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); +with cte1 as (select v3 from t2) select count(*) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1); +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; + +-- dedup the subquery with inner join +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6); +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t3,t2 where v4=v6); -- change the join order,different index in inner join +explain verbose select * from t1 where v1 in (select v3 from t2,t3 where v4=v6); +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + +select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); +select * from t1 where v1 in (select v3 from t2,t3 where v4=v6 and v4 < 10); + +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6); +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from t3,pt2 where v4=v6); +explain verbose select * from pt1 where v1 in (select v3 from pt2,t3 where v4=v6); +explain verbose select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + +select * from pt1 where v1 in (select v3 from pt2) and v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); +select * from pt1 where v1 in (select v3 from pt2,t3 where v4=v6 and v4 < 10); + +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6); +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from t3,cte1 where v4=v6); +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); + +set optimizer_cte_inlining to on; +set optimizer_cte_inlining_bound to 2; +explain verbose with cte1 as (select v3 from t2) select * from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); +with cte1 as (select v3 from t2) select sum(v1) from t1,t2 where t1.v1 in (select v3 from cte1) and t1.v1 in (select v3 from cte1,t3 where v4=v6 and v4 < 10); +reset optimizer_cte_inlining; +reset optimizer_cte_inlining_bound; + +-- dedup the subquery with semi join(still be a scalar in the pre-process) +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); +explain verbose select * from t1 where v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + +select count(*) from t1 where v1 in (select v3 from t2) and v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); +select count(*) from t1 where v1 in (select v3 from t2 where exists (SELECT 1 FROM t3 WHERE v5 = v3)); + +-- inner join different key +explain verbose select v1 from t1 where v1 in (select v3 from t2) and v1 in (select v5 from t2,t3 where v5 = v3); +explain verbose select v1 from t1 where v1 in (select v5 from t2,t3 where v5 = v3); +explain verbose select v1 from t1 where v1 in (select v3 from t2) and v1 in (select v6 from t2,t3 where v5 = v3); -- no dedup, because v6 is not the join key + +select count(v1) from t1 where v1 in (select v3 from t2) and v1 in (select v5 from t2,t3 where v5 = v3); +select count(v1) from t1 where v1 in (select v5 from t2,t3 where v5 = v3); + +-- can't dedup +explain verbose select * from t1 where v1 in (select v3 from t2) and v1 in (select v4 from t2 where v3 < 10); -- different outpt +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 where v3 < 10); -- different scalar ident +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 group by v3); -- group by +explain verbose select * from t1 where v1 in (select v3 from t2) and v2 in (select v3 from t2 order by v3); -- order by/limit, actully this case can be the subset + +reset optimizer_trace_fallback; + +-- start_ignore +drop table if exists t1; +drop table if exists t2; +drop table if exists t3; +drop table if exists pt1; +drop table if exists pt2; +-- end_ignore From f28466789c2fb197fdea43db67e3ac243e7cbb72 Mon Sep 17 00:00:00 2001 From: Jianghua Yang Date: Wed, 18 Jun 2025 23:36:48 +0000 Subject: [PATCH 015/244] gpfdist add --compress-level --- gpMgmt/doc/gpfdist_help | 11 ++++++- src/bin/gpfdist/gpfdist.c | 29 +++++++++++++++---- .../regress/input/gpfdist2_compress.source | 2 +- .../regress/output/gpfdist2_compress.source | 2 +- 4 files changed, 35 insertions(+), 9 deletions(-) diff --git a/gpMgmt/doc/gpfdist_help b/gpMgmt/doc/gpfdist_help index 4ad1f46ce83..cdb298662fc 100755 --- a/gpMgmt/doc/gpfdist_help +++ b/gpMgmt/doc/gpfdist_help @@ -10,7 +10,7 @@ SYNOPSIS gpfdist [-d ] [-p ] [-l ] [-t ] [-S] [-w