From bb93a709dfcd0dfbbb663fa7fe0f224927200634 Mon Sep 17 00:00:00 2001 From: Xing Guo Date: Mon, 31 Jul 2023 20:23:23 +0800 Subject: [PATCH 01/32] Replace pygresql with psycopg2 for gpMgmt tools. (#15988) This is the last patch for replacing pygresql with psycopg2 in Greenplum. This patch mainly targets the gpMgmt tools. Benefits for replacing pygresql with psycopg2. - Psycopg2 is maintained actively we have encountered bugs that haven't been fixed by the upstream yet, e.g., https://github.com/greenplum-db/gpdb/pull/13953. - Psycopg2 is provided by Rocky Linux and Ubuntu. That is to say, we don't need to vendor it ourselves. - Last but not least, we got a chance to clean up leacy codes during the removal process, e.g., https://github.com/greenplum-db/gpdb/pull/15983. After this patch, we need to do the following things. - Add psycopg2 as a dependency of the rpm/deb package. - Remove the pygresql source code tarball from the gpdb repo. - Tidy up READMEs and requirements.txt files. --------- Co-authored-by: Chen Mulong Co-authored-by: Xiaoxiao He Co-authored-by: zhrt123 Co-authored-by: Piyush Chandwadkar Co-authored-by: Praveen Kumar <36772398+kpraveen457@users.noreply.github.com> --- gpMgmt/bin/analyzedb | 17 +- gpMgmt/bin/gpactivatestandby | 5 +- gpMgmt/bin/gpcheckcat | 711 +++++++++--------- .../gpcheckcat_modules/foreign_key_check.py | 39 +- .../leaked_schema_dropper.py | 13 +- .../orphaned_toast_tables_check.py | 8 +- .../unique_index_violation_check.py | 32 +- gpMgmt/bin/gpconfig | 3 +- gpMgmt/bin/gpexpand | 13 +- gpMgmt/bin/gpload.py | 20 +- gpMgmt/bin/gpmovemirrors | 5 +- gpMgmt/bin/gppylib/commands/base.py | 4 +- gpMgmt/bin/gppylib/commands/pg.py | 2 +- .../test/unit/test_unit_pg_base_backup.py | 2 +- gpMgmt/bin/gppylib/db/catalog.py | 4 +- gpMgmt/bin/gppylib/db/dbconn.py | 64 +- .../db/test/unit/test_cluster_dbconn.py | 2 +- gpMgmt/bin/gppylib/gpcatalog.py | 14 +- .../operations/segment_reconfigurer.py | 38 +- .../unit/test_unit_segment_reconfigurer.py | 37 +- .../operations/test/unit/test_unit_utils.py | 2 +- .../gppylib/operations/test_utils_helper.py | 4 +- gpMgmt/bin/gppylib/programs/clsSystemState.py | 6 +- .../test/unit/test_unit_foreign_key_check.py | 13 +- .../gppylib/test/unit/test_unit_gpcheckcat.py | 56 +- .../gppylib/test/unit/test_unit_gpconfig.py | 3 +- .../gppylib/test/unit/test_unit_gpstate.py | 4 +- .../unit/test_unit_leaked_schema_dropper.py | 46 +- .../test_unit_unique_index_violation_check.py | 32 +- gpMgmt/bin/gppylib/utils.py | 13 +- gpMgmt/bin/gpsd | 24 +- gpMgmt/bin/minirepro | 32 +- gpMgmt/sbin/gpsegstop.py | 1 - .../behave/mgmt_utils/steps/mgmt_utils.py | 55 +- .../steps/replication_slots_utils.py | 11 +- gpMgmt/test/behave_utils/gpexpand_dml.py | 50 +- gpMgmt/test/behave_utils/utils.py | 13 +- .../gp_replica_check/gp_replica_check.py | 16 +- python-dependencies.txt | 1 + 39 files changed, 748 insertions(+), 667 deletions(-) diff --git a/gpMgmt/bin/analyzedb b/gpMgmt/bin/analyzedb index a2ad49a7ff30..2a0a0ce81d73 100755 --- a/gpMgmt/bin/analyzedb +++ b/gpMgmt/bin/analyzedb @@ -25,16 +25,15 @@ from contextlib import closing import pipes # for shell-quoting, pipes.quote() import fcntl import itertools - +import psycopg2 try: - import pg - from gppylib import gplog, pgconf, userinput from gppylib.commands.base import Command, WorkerPool, Worker from gppylib.operations import Operation from gppylib.gpversion import GpVersion from gppylib.db import dbconn from gppylib.operations.unix import CheckDir, CheckFile, MakeDir + from gppylib.utils import escape_string except ImportError as e: sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -166,7 +165,7 @@ def validate_schema_exists(pg_port, dbname, schema): try: dburl = dbconn.DbURL(port=pg_port, dbname=dbname) conn = dbconn.connect(dburl) - count = dbconn.querySingleton(conn, "select count(*) from pg_namespace where nspname='%s';" % pg.escape_string(schema)) + count = dbconn.querySingleton(conn, "select count(*) from pg_namespace where nspname='%s';" % escape_string(schema)) if count == 0: raise ExceptionNoStackTraceNeeded("Schema %s does not exist in database %s." % (schema, dbname)) finally: @@ -213,7 +212,7 @@ def get_partition_state_tuples(pg_port, dbname, catalog_schema, partition_info): try: modcount_sql = "select to_char(coalesce(sum(modcount::bigint), 0), '999999999999999999999') from gp_dist_random('%s.%s')" % (catalog_schema, tupletable) modcount = dbconn.querySingleton(conn, modcount_sql) - except pg.DatabaseError as e: + except psycopg2.DatabaseError as e: if "does not exist" in str(e): logger.info("Table %s.%s (%s) no longer exists and will not be analyzed", schemaname, partition_name, tupletable) else: @@ -971,7 +970,7 @@ def get_oid_str(table_list): def regclass_schema_tbl(schema, tbl): schema_tbl = "%s.%s" % (escape_identifier(schema), escape_identifier(tbl)) - return "to_regclass('%s')" % (pg.escape_string(schema_tbl)) + return "to_regclass('%s')" % (escape_string(schema_tbl)) # Escape double-quotes in a string, so that the resulting string is suitable for @@ -1239,7 +1238,7 @@ def validate_tables(conn, tablenames): while curr_batch < nbatches: batch = tablenames[curr_batch * batch_size:(curr_batch + 1) * batch_size] - oid_str = ','.join(map((lambda x: "('%s')" % pg.escape_string(x)), batch)) + oid_str = ','.join(map((lambda x: "('%s')" % escape_string(x)), batch)) if not oid_str: break @@ -1255,7 +1254,7 @@ def get_include_cols_from_exclude(conn, schema, table, exclude_cols): """ Given a list of excluded columns of a table, get the list of included columns """ - quoted_exclude_cols = ','.join(["'%s'" % pg.escape_string(x) for x in exclude_cols]) + quoted_exclude_cols = ','.join(["'%s'" % escape_string(x) for x in exclude_cols]) oid_str = regclass_schema_tbl(schema, table) cols = run_sql(conn, GET_INCLUDED_COLUMNS_FROM_EXCLUDE_SQL % (oid_str, quoted_exclude_cols)) @@ -1271,7 +1270,7 @@ def validate_columns(conn, schema, table, column_list): return sql = VALIDATE_COLUMN_NAMES_SQL % (regclass_schema_tbl(schema, table), - ','.join(["'%s'" % pg.escape_string(x) for x in column_list])) + ','.join(["'%s'" % escape_string(x) for x in column_list])) valid_col_count = dbconn.querySingleton(conn, sql) if int(valid_col_count) != len(column_list): diff --git a/gpMgmt/bin/gpactivatestandby b/gpMgmt/bin/gpactivatestandby index a38ec8cb00ac..f65b9bdcc825 100755 --- a/gpMgmt/bin/gpactivatestandby +++ b/gpMgmt/bin/gpactivatestandby @@ -21,10 +21,9 @@ import time import shutil import tempfile from datetime import datetime, timedelta - +import psycopg2 # import GPDB modules try: - import pg as pygresql from gppylib.commands import unix, gp, pg from gppylib.db import dbconn from gppylib.gpparseopts import OptParser, OptChecker, OptionGroup, SUPPRESS_HELP @@ -341,7 +340,7 @@ def promote_standby(coordinator_data_dir): dbconn.execSQL(conn, 'CHECKPOINT') conn.close() return True - except pygresql.InternalError as e: + except (psycopg2.InternalError, psycopg2.OperationalError) as e: pass time.sleep(1) diff --git a/gpMgmt/bin/gpcheckcat b/gpMgmt/bin/gpcheckcat index 1b047bafa0b0..ed918a223ab8 100755 --- a/gpMgmt/bin/gpcheckcat +++ b/gpMgmt/bin/gpcheckcat @@ -27,23 +27,20 @@ import re import sys import time from functools import reduce - +import psycopg2 +from psycopg2 import extras +from contextlib import closing try: from gppylib import gplog - from gppylib.db import dbconn from gppylib.gpcatalog import * from gppylib.commands.unix import * from gppylib.commands.gp import conflict_with_gpexpand from gppylib.system.info import * - from pgdb import DatabaseError from gpcheckcat_modules.unique_index_violation_check import UniqueIndexViolationCheck from gpcheckcat_modules.leaked_schema_dropper import LeakedSchemaDropper from gpcheckcat_modules.repair import Repair from gpcheckcat_modules.foreign_key_check import ForeignKeyCheck from gpcheckcat_modules.orphaned_toast_tables_check import OrphanedToastTablesCheck - - import pg - except ImportError as e: sys.exit('Error: unable to import module: ' + str(e)) @@ -136,7 +133,7 @@ class Global(): self.dbname = None self.firstdb = None self.alldb = [] - self.db = {} + self.conn = {} self.tmpdir = None self.reset_stmt_queues() @@ -205,30 +202,27 @@ def usage(exitarg=None): ############################### def getversion(): - db = connect() - curs = db.query(''' - select regexp_replace(version(), - E'.*PostgreSQL [^ ]+ .Greenplum Database ([1-9]+.[0-9]+|main).*', - E'\\\\1') as ver;''') - - row = curs.getresult()[0] - version = row[0] - - logger.debug('got version %s' % version) - return version - + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute(''' + select regexp_replace(version(), + E'.*PostgreSQL [^ ]+ .Greenplum Database ([1-9]+.[0-9]+|main).*', + E'\\\\1') as ver;''') + row = curs.fetchone() + version = row[0] + logger.debug('got version %s' % version) + return version ############################### def getalldbs(): """ get all connectable databases """ - db = connect() - curs = db.query(''' - select datname from pg_database where datallowconn order by datname ''') - row = curs.getresult() - return row - + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute('''select datname from pg_database where datallowconn order by datname''') + row = curs.fetchall() + return row ############################### def parseCommandLine(): @@ -337,19 +331,21 @@ def connect(user=None, password=None, host=None, port=None, try: logger.debug('connecting to %s:%s %s' % (host, port, database)) - db = pg.connect(host=host, port=port, user=user, - passwd=password, dbname=database, opt=options) - - except pg.InternalError as ex: + conn = psycopg2.connect(host=host, port=port, user=user, + password=password, dbname=database, options=options) + ## Don't execute query in a transaction block. + conn.set_session(autocommit=True) + except (psycopg2.InternalError, psycopg2.OperationalError) as ex: logger.fatal('could not connect to %s: "%s"' % (database, str(ex).strip())) exit(1) logger.debug('connected with %s:%s %s' % (host, port, database)) - return db + return conn -############# +# NOTE: We cannot use connect2() with contextmanager, since we manage the connection +# ourselves. def connect2(cfgrec, user=None, password=None, database=None, utilityMode=True): host = cfgrec['address'] port = cfgrec['port'] @@ -361,22 +357,22 @@ def connect2(cfgrec, user=None, password=None, database=None, utilityMode=True): key = "%s.%s.%s.%s.%s.%s.%s" % (host, port, datadir, user, password, database, str(utilityMode)) - conns = GV.db.get(key) + conns = GV.conn.get(key) if conns: return conns[0] conn = connect(host=host, port=port, user=user, password=password, database=database, utilityMode=utilityMode) if conn: - GV.db[key] = [conn, cfgrec] + GV.conn[key] = [conn, cfgrec] return conn class execThread(Thread): - def __init__(self, cfg, db, qry): + def __init__(self, cfg, conn, qry): self.cfg = cfg - self.db = db + self.conn = conn self.qry = qry self.curs = None self.error = None @@ -384,11 +380,11 @@ class execThread(Thread): def run(self): try: - self.curs = self.db.query(self.qry) + self.curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + self.curs.execute(self.qry) except BaseException as e: self.error = e - def processThread(threads): batch = [] for th in threads: @@ -417,9 +413,9 @@ def connect2run(qry, col=None): # parallelise queries for dbid in GV.cfg: c = GV.cfg[dbid] - db = connect2(c) + conn = connect2(c) - thread = execThread(c, db, qry) + thread = execThread(c, conn, qry) thread.start() logger.debug('launching query thread %s for dbid %i' % (thread.name, dbid)) @@ -439,8 +435,8 @@ def connect2run(qry, col=None): err = [] for [cfg, curs] in batch: if col is None: - col = curs.listfields() - for row in curs.dictresult(): + col = [desc[0] for desc in curs.description] + for row in curs.fetchall(): err.append([cfg, col, row]) return err @@ -458,7 +454,6 @@ def formatErr(c, col, row): ############# def getGPConfiguration(): cfg = {} - db = connect() # note that in 4.0, sql commands cannot be run against the segment mirrors directly # so we filter out non-primary segment databases in the query qry = ''' @@ -468,13 +463,14 @@ def getGPConfiguration(): FROM gp_segment_configuration WHERE (role = 'p' or content < 0 ) ''' - curs = db.query(qry) - for row in curs.dictresult(): - if row['content'] == -1 and not row['isprimary']: - continue # skip standby coordinator - cfg[row['dbid']] = row - db.close() - return cfg + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + for row in curs.fetchall(): + if row['content'] == -1 and not row['isprimary']: + continue # skip standby coordinator + cfg[row['dbid']] = row + return cfg def checkDistribPolicy(): logger.info('-----------------------------------') @@ -488,27 +484,27 @@ def checkDistribPolicy(): where pk.contype in('p', 'u') and d.policytype = 'p' and d.distkey = '' ''' - db = connect2(GV.cfg[GV.coordinator_dbid]) try: - curs = db.query(qry) - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) - - if not err: - logger.info('[OK] randomly distributed tables') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] randomly distributed tables') - logger.error('pg_constraint has %d issue(s)' % len(err)) - logger.error(qry) - for e in err: - logger.error(formatErr(e[0], e[1], e[2])) - for e in err: - cons = e[2] - removeIndexConstraint(cons['nspname'], cons['relname'], - cons['constraint']) + conn = connect2(GV.cfg[GV.coordinator_dbid]) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) + + if not err: + logger.info('[OK] randomly distributed tables') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] randomly distributed tables') + logger.error('pg_constraint has %d issue(s)' % len(err)) + logger.error(qry) + for e in err: + logger.error(formatErr(e[0], e[1], e[2])) + cons = e[2] + removeIndexConstraint(cons['nspname'], cons['relname'], + cons['constraint']) except Exception as e: setError(ERROR_NOREPAIR) myprint('[ERROR] executing test: checkDistribPolicy') @@ -530,22 +526,23 @@ def checkDistribPolicy(): and not d.distkey::int2[] operator(pg_catalog.<@) pk.conkey ''' try: - curs = db.query(qry) - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) - - if not err: - logger.info('[OK] unique constraints') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] unique constraints') - logger.error('pg_constraint has %d issue(s)' % len(err)) - logger.error(qry) - for e in err: logger.error(formatErr(e[0], e[1], e[2])) + conn = connect2(GV.cfg[GV.coordinator_dbid]) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + err = [] + curs.execute(qry) + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) + + if not err: + logger.info('[OK] unique constraints') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] unique constraints') + logger.error('pg_constraint has %d issue(s)' % len(err)) + logger.error(qry) for e in err: + logger.error(formatErr(e[0], e[1], e[2])) cons = e[2] removeIndexConstraint(cons['nspname'], cons['relname'], cons['constraint']) @@ -561,7 +558,6 @@ def checkPartitionIntegrity(): logger.info('-----------------------------------') logger.info('Checking pg_partition ...') err = [] - db = connect() # Check for the numsegments value of parent and child partition from the gp_distribution_policy table qry = ''' @@ -573,53 +569,55 @@ def checkPartitionIntegrity(): and not (inhrelid in (select ftrelid from pg_catalog.pg_foreign_table) and child.numsegments = NULL); ''' try: - curs = db.query(qry) - cols = ('inhparent', 'inhrelid', 'numsegments_parent', 'numsegments_child') - col_names = { - 'inhparent': 'table', - 'inhrelid': 'affected child', - 'numsegments_parent': 'parent numsegments value', - 'numsegments_child': 'child numsegments value', - } - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], cols, row]) - - if not err: - logger.info('[OK] partition numsegments check') - else: - err_count = len(err) - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] partition numsegments check') - logger.error('partition numsegments check found %d issue(s)' % err_count) - if err_count > 100: - logger.error(qry) + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + cols = ('inhparent', 'inhrelid', 'numsegments_parent', 'numsegments_child') + col_names = { + 'inhparent': 'table', + 'inhrelid': 'affected child', + 'numsegments_parent': 'parent numsegments value', + 'numsegments_child': 'child numsegments value', + } + + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], cols, row]) + + if not err: + logger.info('[OK] partition numsegments check') + else: + err_count = len(err) + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] partition numsegments check') + logger.error('partition numsegments check found %d issue(s)' % err_count) + if err_count > 100: + logger.error(qry) - myprint( - '[ERROR]: child partition(s) have different numsegments value ' - 'from the root partition. Check the gpcheckcat log for details.' - ) - logger.error('The following tables have different numsegments value (showing at most 100 rows):') + myprint( + '[ERROR]: child partition(s) have different numsegments value ' + 'from the root partition. Check the gpcheckcat log for details.' + ) + logger.error('The following tables have different numsegments value (showing at most 100 rows):') - # report at most 100 rows, for brevity - err = err[:100] + # report at most 100 rows, for brevity + err = err[:100] - for index, e in enumerate(err): - cfg = e[0] - col = e[1] - row = e[2] + for index, e in enumerate(err): + cfg = e[0] + col = e[1] + row = e[2] - if index == 0: - logger.error("--------") - logger.error(" " + " | ".join(map(col_names.get, col))) - logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) + if index == 0: + logger.error("--------") + logger.error(" " + " | ".join(map(col_names.get, col))) + logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) - logger.error(" " + " | ".join([str(row[x]) for x in col])) + logger.error(" " + " | ".join([str(row[x]) for x in col])) - if err_count > 100: - logger.error(" ...") + if err_count > 100: + logger.error(" ...") except Exception as e: setError(ERROR_NOREPAIR) @@ -642,74 +640,74 @@ def checkPartitionIntegrity(): and (select isleaf from pg_partition_tree(inhparent) where relid = inhrelid)); ''' try: - curs = db.query(qry) - cols = ('inhparent', 'inhrelid', 'dby_parent', 'dby_child') - col_names = { - 'inhparent': 'table', - 'inhrelid': 'affected child', - 'dby_parent': 'table distribution key', - 'dby_child': 'child distribution key', - } - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], cols, row]) - - if not err: - logger.info('[OK] partition distribution policy check') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] partition distribution policy check') - logger.error('partition distribution policy check found %d issue(s)' % len(err)) - if len(err) > 100: - logger.error(qry) - - myprint( - '[ERROR]: child partition(s) are distributed differently from ' - 'the root partition, and must be manually redistributed, for ' - 'some tables. Check the gpcheckcat log for details.' - ) - logger.error('The following tables must be manually redistributed:') - - count = 0 - for e in err: - cfg = e[0] - col = e[1] - row = e[2] - - # TODO: generate a repair script for this row. This is - # difficult, since we can't redistribute child partitions - # directly. - - # report at most 100 rows, for brevity - if count == 100: - logger.error("...") - count += 1 - if count > 100: - continue - - if count == 0: - logger.error("--------") - logger.error(" " + " | ".join(map(col_names.get, col))) - logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) - - logger.error(" " + " | ".join([str(row[x]) for x in col])) - count += 1 - - logger.error( - 'Execute an ALTER TABLE ... SET DISTRIBUTED BY statement, with ' - 'the desired distribution key, on the partition root for each ' - 'affected table.' - ) + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + cols = ('inhparent', 'inhrelid', 'dby_parent', 'dby_child') + col_names = { + 'inhparent': 'table', + 'inhrelid': 'affected child', + 'dby_parent': 'table distribution key', + 'dby_child': 'child distribution key', + } + + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], cols, row]) + + if not err: + logger.info('[OK] partition distribution policy check') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] partition distribution policy check') + logger.error('partition distribution policy check found %d issue(s)' % len(err)) + if len(err) > 100: + logger.error(qry) + + myprint( + '[ERROR]: child partition(s) are distributed differently from ' + 'the root partition, and must be manually redistributed, for ' + 'some tables. Check the gpcheckcat log for details.' + ) + logger.error('The following tables must be manually redistributed:') + + count = 0 + for e in err: + cfg = e[0] + col = e[1] + row = e[2] + + # TODO: generate a repair script for this row. This is + # difficult, since we can't redistribute child partitions + # directly. + + # report at most 100 rows, for brevity + if count == 100: + logger.error("...") + count += 1 + if count > 100: + continue + + if count == 0: + logger.error("--------") + logger.error(" " + " | ".join(map(col_names.get, col))) + logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) + + logger.error(" " + " | ".join([str(row[x]) for x in col])) + count += 1 + + logger.error( + 'Execute an ALTER TABLE ... SET DISTRIBUTED BY statement, with ' + 'the desired distribution key, on the partition root for each ' + 'affected table.' + ) except Exception as e: setError(ERROR_NOREPAIR) myprint('[ERROR] executing test: checkPartitionIntegrity') myprint(' Execution error: ' + str(e)) - db.close() - checkPoliciesRepair() ############# @@ -776,7 +774,7 @@ Produce repair scripts to remove dangling entries of gp_fastsequence: ''' -def removeFastSequence(db): +def removeFastSequence(conn): ''' MPP-14758: gp_fastsequence does not get cleanup after a failed transaction (AO/CO) Note: this is slightly different from the normal foreign key check @@ -802,14 +800,15 @@ def removeFastSequence(db): ON r.gp_segment_id = cfg.content WHERE cfg.role = 'p'; """ - curs = db.query(qry) - for row in curs.dictresult(): - seg = row['dbid'] # dbid of targeted segment - name = 'gp_fastsequence tuple' # for comment purposes - table = 'gp_fastsequence' # table name - cols = {'objid': row['objid']} # column name and value - objname = 'gp_fastsequence' # for comment purposes - buildRemove(seg, name, table, cols, objname) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + for row in curs.fetchall(): + seg = row['dbid'] # dbid of targeted segment + name = 'gp_fastsequence tuple' # for comment purposes + table = 'gp_fastsequence' # table name + cols = {'objid': row['objid']} # column name and value + objname = 'gp_fastsequence' # for comment purposes + buildRemove(seg, name, table, cols, objname) except Exception as e: logger.error('removeFastSequence: ' + str(e)) @@ -871,42 +870,41 @@ def drop_leaked_schemas(leaked_schema_dropper, dbname): logger.info('-----------------------------------') logger.info('Checking for leaked temporary schemas') - db_connection = connect(database=dbname) try: - dropped_schemas = leaked_schema_dropper.drop_leaked_schemas(db_connection) - if not dropped_schemas: - logger.info('[OK] temporary schemas') - else: - logger.info('[FAIL] temporary schemas') - myprint("Found and dropped %d unbound temporary schemas" % len(dropped_schemas)) - logger.error('Dropped leaked schemas \'%s\' in the database \'%s\'' % (dropped_schemas, dbname)) + with closing(connect(database=dbname)) as conn: + dropped_schemas = leaked_schema_dropper.drop_leaked_schemas(conn) + if not dropped_schemas: + logger.info('[OK] temporary schemas') + else: + logger.info('[FAIL] temporary schemas') + myprint("Found and dropped %d unbound temporary schemas" % len(dropped_schemas)) + logger.error('Dropped leaked schemas \'%s\' in the database \'%s\'' % (dropped_schemas, dbname)) except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) - finally: - db_connection.close() def checkDepend(): # Check for dependencies on non-existent objects logger.info('-----------------------------------') logger.info('Checking Object Dependencies') - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: - # Catalogs that link up to pg_depend/pg_shdepend - qry = """ - select relname from pg_class c - where relkind='r' - and relnamespace=%d - and exists (select 1 from pg_attribute a where attname = 'oid' and a.attrelid = c.oid) - """ % PG_CATALOG_OID - curs = db.query(qry) - catalogs = [] - for row in curs.getresult(): - catalogs.append(row[0]) - - checkDependJoinCatalog(catalogs) - checkCatalogJoinDepend(catalogs) + # Catalogs that link up to pg_depend/pg_shdepend + qry = """ + select relname from pg_class c + where relkind='r' + and relnamespace=%d + and exists (select 1 from pg_attribute a where attname = 'oid' and a.attrelid = c.oid) + """ % PG_CATALOG_OID + curs.execute(qry) + catalogs = [] + for row in curs.fetchall(): + catalogs.append(row[0]) + + checkDependJoinCatalog(catalogs) + checkCatalogJoinDepend(catalogs) def checkDependJoinCatalog(catalogs): # Construct subquery that will verify that all (classid, objid) @@ -1063,7 +1061,6 @@ def checkOwners(): # # - Between 3.3 and 4.0 the ao segment columns migrated from pg_class # to pg_appendonly. - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) qry = ''' select distinct n.nspname, coalesce(o.relname, c.relname) as relname, a.rolname, m.rolname as coordinator_rolname @@ -1080,20 +1077,21 @@ def checkOwners(): where c.relowner <> r.relowner ''' try: - curs = db.query(qry) - - rows = [] - for row in curs.dictresult(): - rows.append(row) - - if len(rows) == 0: - logger.info('[OK] table ownership') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] table ownership') - logger.error('found %d table ownership issue(s)' % len(rows)) - logger.error('%s' % qry) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + rows = [] + for row in curs.fetchall(): + rows.append(row) + + if len(rows) == 0: + logger.info('[OK] table ownership') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] table ownership') + logger.error('found %d table ownership issue(s)' % len(rows)) + logger.error('%s' % qry) for row in rows[0:100]: logger.error(' %s.%s relowner %s != %s' % (row['nspname'], row['relname'], row['rolname'], @@ -1117,7 +1115,6 @@ def checkOwners(): # - Ignore implementation types of pg_class entries - they should be # in the check above since ALTER TABLE is required to fix them, not # ALTER TYPE. - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) qry = ''' select distinct n.nspname, t.typname, a.rolname, m.rolname as coordinator_rolname from gp_dist_random('pg_type') r @@ -1128,27 +1125,28 @@ def checkOwners(): where r.typowner <> t.typowner ''' try: - curs = db.query(qry) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) - rows = [] - for row in curs.dictresult(): - rows.append(row) + rows = [] + for row in curs.fetchall(): + rows.append(row) - if len(rows) == 0: - logger.info('[OK] type ownership') - else: - GV.checkStatus = False - setError(ERROR_NOREPAIR) - logger.info('[FAIL] type ownership') - logger.error('found %d type ownership issue(s)' % len(rows)) - logger.error('%s' % qry) + if len(rows) == 0: + logger.info('[OK] type ownership') + else: + GV.checkStatus = False + setError(ERROR_NOREPAIR) + logger.info('[FAIL] type ownership') + logger.error('found %d type ownership issue(s)' % len(rows)) + logger.error('%s' % qry) for row in rows[0:100]: logger.error(' %s.%s typeowner %s != %s' % (row['nspname'], row['typname'], row['rolname'], row['coordinator_rolname'])) if len(rows) > 100: logger.error("...") - except Exception as e: setError(ERROR_NOREPAIR) myprint("[ERROR] executing test: check type ownership") @@ -1172,15 +1170,14 @@ def checkOwners(): def closeDbs(): - for key, conns in GV.db.items(): - db = conns[0] - db.close() - GV.db = {} # remove everything + for key, conns in GV.conn.items(): + conn = conns[0] + conn.close() + GV.conn = {} # remove everything # ------------------------------------------------------------------------------- def getCatObj(namestr): - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) try: cat = GV.catalog.getCatalogTable(namestr) except Exception as e: @@ -1250,25 +1247,25 @@ def checkTableACL(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() - - if nrows == 0: - logger.info('[OK] Cross consistency acl check for ' + catname) - else: - GV.checkStatus = False - setError(ERROR_NOREPAIR) - GV.aclStatus = False - logger.info('[FAIL] Cross consistency acl check for ' + catname) - logger.error(' %s acl check has %d issue(s)' % (catname, nrows)) - - fields = curs.listfields() - gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): - gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) - processACLResult(catname, fields, curs.getresult()) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: + curs = db.execute(qry) + nrows = curs.rowcount + if nrows == 0: + logger.info('[OK] Cross consistency acl check for ' + catname) + else: + GV.checkStatus = False + setError(ERROR_NOREPAIR) + GV.aclStatus = False + logger.info('[FAIL] Cross consistency acl check for ' + catname) + logger.error(' %s acl check has %d issue(s)' % (catname, nrows)) + + fields = [desc[0] for desc in curs.description] + gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) + for row in curs.getresult(): + gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) + processACLResult(catname, fields, curs.getresult()) except Exception as e: setError(ERROR_NOREPAIR) GV.aclStatus = False @@ -1289,9 +1286,9 @@ def checkForeignKey(cat_tables=None): if not cat_tables: cat_tables = GV.catalog.getCatalogTables() - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) try: - foreign_key_check = ForeignKeyCheck(db_connection, logger, GV.opt['-S'], autoCast) + foreign_key_check = ForeignKeyCheck(conn, logger, GV.opt['-S'], autoCast) foreign_key_issues = foreign_key_check.runCheck(cat_tables) if foreign_key_issues: GV.checkStatus = False @@ -1301,13 +1298,14 @@ def checkForeignKey(cat_tables=None): processForeignKeyResult(catname, pkcatname, fields, results) if catname == 'gp_fastsequence' and pkcatname == 'pg_class': setError(ERROR_REMOVE) - removeFastSequence(db_connection) + removeFastSequence(conn) else: setError(ERROR_NOREPAIR) except Exception as ex: setError(ERROR_NOREPAIR) GV.foreignKeyStatus = False myprint(' Execution error: ' + str(ex)) + # ------------------------------------------------------------------------------- @@ -1377,40 +1375,39 @@ def checkTableMissingEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() - results = curs.getresult() - fields = curs.listfields() - - if nrows != 0: - results = filterSpuriousFailures(catname, fields, results) - nrows = len(results) - - if nrows == 0: - logger.info('[OK] Checking for missing or extraneous entries for ' + catname) - else: - if catname in ['pg_constraint']: - logger_with_level = logger.warning - log_level = logging.WARNING + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: + curs.execute(qry) + nrows = curs.rowcount + results = curs.fetchall() + fields = [desc[0] for desc in curs.description] + + if nrows != 0: + results = filterSpuriousFailures(catname, fields, results) + nrows = len(results) + + if nrows == 0: + logger.info('[OK] Checking for missing or extraneous entries for ' + catname) else: - GV.checkStatus = False - GV.missingEntryStatus = False - logger_with_level = logger.error - log_level = logging.ERROR - - logger.info(('[%s] Checking for missing or extraneous entries for ' + catname) % - ('WARNING' if log_level == logging.WARNING else 'FAIL')) - logger_with_level(' %s has %d issue(s)' % (catname, nrows)) - gplog.log_literal(logger, log_level, " " + " | ".join(fields)) + if catname in ['pg_constraint']: + logger_with_level = logger.warning + log_level = logging.WARNING + else: + GV.checkStatus = False + GV.missingEntryStatus = False + logger_with_level = logger.error + log_level = logging.ERROR + + logger.info(('[%s] Checking for missing or extraneous entries for ' + catname) % + ('WARNING' if log_level == logging.WARNING else 'FAIL')) + logger_with_level(' %s has %d issue(s)' % (catname, nrows)) + gplog.log_literal(logger, log_level, " " + " | ".join(fields)) for row in results: gplog.log_literal(logger, log_level, " " + " | ".join(map(str, row))) processMissingDuplicateEntryResult(catname, fields, results, "missing") if catname == 'pg_type': generateVerifyFile(catname, fields, results, 'missing_extraneous') - return results - except Exception as e: setError(ERROR_NOREPAIR) GV.missingEntryStatus = False @@ -1420,8 +1417,8 @@ def checkTableMissingEntry(cat): class checkAOSegVpinfoThread(execThread): - def __init__(self, cfg, db): - execThread.__init__(self, cfg, db, None) + def __init__(self, cfg, conn): + execThread.__init__(self, cfg, conn, None) def run(self): aoseg_query = """ @@ -1432,15 +1429,16 @@ class checkAOSegVpinfoThread(execThread): try: # Read the list of aoseg tables from the database - curs = self.db.query(aoseg_query) + curs = self.conn.cursor() + curs.execute(aoseg_query) - for relname, relid, segrelid, segrelname in curs.getresult(): + for relname, relid, segrelid, segrelname in curs.fetchall(): qry = "SELECT count(*) FROM pg_attribute WHERE attrelid=%d AND attnum > 0;" % (relid) attr_count = self.db.query(qry).getresult()[0][0] qry = "SELECT distinct(length(vpinfo)) FROM pg_aoseg.%s where state = 1;" % (segrelname) - vpinfo_curs = self.db.query(qry) - nrows = vpinfo_curs.ntuples() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: continue elif nrows > 1: @@ -1455,7 +1453,7 @@ class checkAOSegVpinfoThread(execThread): logger.error(qry) continue - vpinfo_length = vpinfo_curs.getresult()[0][0] + vpinfo_length = curs.fetchone()[0] # vpinfo is bytea type, the length of the first 3 fields is 12 bytes, and the size of AOCSVPInfoEntry is 16 # typedef struct AOCSVPInfo @@ -1488,8 +1486,8 @@ def checkAOSegVpinfo(): # parallelise check for dbid in GV.cfg: cfg = GV.cfg[dbid] - db_connection = connect2(cfg) - thread = checkAOSegVpinfoThread(cfg, db_connection) + conn = connect2(cfg) + thread = checkAOSegVpinfoThread(cfg, conn) thread.start() logger.debug('launching check thread %s for dbid %i' % (thread.name, dbid)) @@ -1617,9 +1615,10 @@ def checkTableInconsistentEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: logger.info('[OK] Checking for inconsistent entries for ' + catname) @@ -1630,16 +1629,14 @@ def checkTableInconsistentEntry(cat): logger.info('[FAIL] Checking for inconsistent entries for ' + catname) logger.error(' %s has %d issue(s)' % (catname, nrows)) - fields = curs.listfields() + fields = [desc[0] for desc in curs.description] gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): + results = curs.fetchall() + for row in results: gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) - results = curs.getresult() processInconsistentEntryResult(catname, pkey, fields, results) if catname == 'pg_type': generateVerifyFile(catname, fields, results, 'duplicate') - - except Exception as e: setError(ERROR_NOREPAIR) GV.inconsistentEntryStatus = False @@ -1753,9 +1750,10 @@ def checkTableDuplicateEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: logger.info('[OK] Checking for duplicate entries for ' + catname) @@ -1767,7 +1765,7 @@ def checkTableDuplicateEntry(cat): fields = curs.listfields() gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - results = curs.getresult() + results = curs.fetchall() for row in results: gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) processMissingDuplicateEntryResult(catname, fields, results, "duplicate") @@ -1816,9 +1814,9 @@ def duplicateEntryQuery(catname, pkey): def checkUniqueIndexViolation(): logger.info('-----------------------------------') logger.info('Performing check: checking for violated unique indexes') - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - violations = UniqueIndexViolationCheck().runCheck(db_connection) + violations = UniqueIndexViolationCheck().runCheck(conn) checkname = 'unique index violation(s)' if violations: @@ -1855,9 +1853,9 @@ def checkOrphanedToastTables(): logger.info('-----------------------------------') logger.info('Performing check: checking for orphaned TOAST tables') - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) checker = OrphanedToastTablesCheck() - check_passed = checker.runCheck(db_connection) + check_passed = checker.runCheck(conn) checkname = 'orphaned toast table(s)' if check_passed: @@ -2376,12 +2374,14 @@ def getOidFromPK(catname, pkeys): pkeystr=pkeystr) try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - if (len(curs.dictresult()) == 0): + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute(qry) + results = curs.fetchall() + if (len(results) == 0): raise QueryException("No such entry '%s' in %s" % (pkeystr, catname)) - return curs.dictresult().pop()['oid'] + return results.pop()['oid'] except Exception as e: setError(ERROR_NOREPAIR) @@ -2393,10 +2393,11 @@ def getOidFromPK(catname, pkeys): def getClassOidForRelfilenode(relfilenode): qry = "SELECT oid FROM pg_class WHERE relfilenode = %d;" % (relfilenode) try: - dburl = dbconn.DbURL(hostname=GV.opt['-h'], port=GV.opt['-p'], dbname=GV.dbname) - conn = dbconn.connect(dburl) - oid = dbconn.queryRow(conn, qry)[0] - return oid + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute(qry) + oid = curs.fetchone()[0] + return oid except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) @@ -2416,10 +2417,12 @@ def getResourceTypeOid(oid): """ % (oid, oid) try: - db = connect() - curs = db.query(qry) - if len(curs.dictresult()) == 0: return 0 - return curs.dictresult().pop()['oid'] + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + results = curs.fetchall() + if len(results) == 0: return 0 + return results.pop()['oid'] except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) @@ -3014,7 +3017,7 @@ class GPObject: # Collect all tables with missing issues for later reporting if len(self.missingIssues): - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) oid_query = "select (select nspname from pg_namespace where oid=relnamespace) || '.' || relname from pg_class where oid=%d" type_query = "select (select nspname from pg_namespace where oid=relnamespace) || '.' || relname from pg_class where reltype=%d" for issues in self.missingIssues.values() : @@ -3022,19 +3025,23 @@ class GPObject: # Get schemaname.tablename corresponding to oid for key in issue.pkeys: if 'relid' in key or key in ['ev_class', 'reloid']: - table_list = db.query(oid_query % issue.pkeys[key]).getresult() + curs = conn.cursor() + curs.execute(oid_query % issue.pkeys[key]) + table_list = curs.fetchone() if table_list: if issue.type == 'missing': - GV.missing_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.missing_attr_tables.append( (table_list[0], issue.segids) ) else: - GV.extra_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.extra_attr_tables.append( (table_list[0], issue.segids) ) elif key == 'oid': - table_list = db.query(type_query % issue.pkeys[key]).getresult() + curs = conn.cursor() + curs.execute(type_query % issue.pkeys[key]) + table_list = curs.fetchone() if table_list: if issue.type == 'missing': - GV.missing_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.missing_attr_tables.append( (table_list[0], issue.segids) ) else: - GV.extra_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.extra_attr_tables.append( (table_list[0], issue.segids) ) def __cmp__(self, other): @@ -3183,9 +3190,11 @@ def getRelInfo(objects): """.format(oids=','.join(map(str, oids))) try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - for row in curs.getresult(): + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + results = curs.fetchall() + for row in results: (oid, relname, nspname, relkind, paroid) = row objects[oid, 'pg_class'].setRelInfo(relname, nspname, relkind, paroid) diff --git a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py index b584b9f7557c..eee4b33af2b1 100644 --- a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py @@ -2,6 +2,7 @@ from gppylib.gplog import * from gppylib.gpcatalog import * +from contextlib import closing import re class ForeignKeyCheck: @@ -116,25 +117,25 @@ def checkTableForeignKey(self, cat): def _validate_relation(self, catname, fkeystr, pkcatname, pkeystr, qry): issue_list = [] try: - curs = self.db_connection.query(qry) - nrows = curs.ntuples() - - if nrows == 0: - self.logger.info('[OK] Foreign key check for %s(%s) referencing %s(%s)' % - (catname, fkeystr, pkcatname, pkeystr)) - else: - self.logger.info('[FAIL] Foreign key check for %s(%s) referencing %s(%s)' % - (catname, fkeystr, pkcatname, pkeystr)) - self.logger.error(' %s has %d issue(s): entry has NULL reference of %s(%s)' % - (catname, nrows, pkcatname, pkeystr)) - - fields = curs.listfields() - log_literal(self.logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): - log_literal(self.logger, logging.ERROR, " " + " | ".join(map(str, row))) - results = curs.getresult() - issue_list.append((pkcatname, fields, results)) - + with closing(self.db_connection.cursor()) as curs: + curs.execute(qry) + nrows = curs.rowcount + + if nrows == 0: + self.logger.info('[OK] Foreign key check for %s(%s) referencing %s(%s)' % + (catname, fkeystr, pkcatname, pkeystr)) + else: + self.logger.info('[FAIL] Foreign key check for %s(%s) referencing %s(%s)' % + (catname, fkeystr, pkcatname, pkeystr)) + self.logger.error(' %s has %d issue(s): entry has NULL reference of %s(%s)' % + (catname, nrows, pkcatname, pkeystr)) + + fields = [desc[0] for desc in curs.description] + log_literal(self.logger, logging.ERROR, " " + " | ".join(fields)) + results = curs.fetchall() + for row in results: + log_literal(self.logger, logging.ERROR, " " + " | ".join(map(str, row))) + issue_list.append((pkcatname, fields, results)) except Exception as e: err_msg = '[ERROR] executing: Foreign key check for catalog table {0}. Query : \n {1}\n'.format(catname, qry) err_msg += str(e) diff --git a/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py b/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py index 87e55a5cf7b7..dc7cfacb32f2 100644 --- a/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py +++ b/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py @@ -35,16 +35,19 @@ class LeakedSchemaDropper: """ def __get_leaked_schemas(self, db_connection): - leaked_schemas = db_connection.query(self.leaked_schema_query) + with db_connection.cursor() as curs: + curs.execute(self.leaked_schema_query) + leaked_schemas = curs.fetchall() - if not leaked_schemas: - return [] + if not leaked_schemas: + return [] - return [row[0] for row in leaked_schemas.getresult() if row[0]] + return [row[0] for row in leaked_schemas if row[0]] def drop_leaked_schemas(self, db_connection): leaked_schemas = self.__get_leaked_schemas(db_connection) for leaked_schema in leaked_schemas: escaped_schema_name = escapeDoubleQuoteInSQLString(leaked_schema) - db_connection.query('DROP SCHEMA IF EXISTS %s CASCADE;' % (escaped_schema_name)) + with db_connection.cursor() as curs: + curs.execute('DROP SCHEMA IF EXISTS %s CASCADE;' % (escaped_schema_name)) return leaked_schemas diff --git a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py index 21ec8d18047e..a76ef5608672 100644 --- a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py @@ -4,6 +4,8 @@ from collections import namedtuple from gpcheckcat_modules.orphan_toast_table_issues import OrphanToastTableIssue, DoubleOrphanToastTableIssue, ReferenceOrphanToastTableIssue, DependencyOrphanToastTableIssue, MismatchOrphanToastTableIssue +import psycopg2 +from psycopg2 import extras OrphanedTable = namedtuple('OrphanedTable', 'oid catname') @@ -117,8 +119,10 @@ def __init__(self): """ def runCheck(self, db_connection): - orphaned_toast_tables = db_connection.query(self.orphaned_toast_tables_query).dictresult() - if len(orphaned_toast_tables) == 0: + curs = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute(self.orphaned_toast_tables_query) + orphaned_toast_tables = curs.fetchall() + if curs.rowcount == 0: return True for row in orphaned_toast_tables: diff --git a/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py b/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py index 47999f5c59c3..6778401f31ac 100644 --- a/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py @@ -34,22 +34,24 @@ def __init__(self): ) as violations """ - def runCheck(self, db_connection): - unique_indexes = db_connection.query(self.unique_indexes_query).getresult() - violations = [] + def runCheck(self, conn): + with conn.cursor() as cur: + cur.execute(self.unique_indexes_query) + unique_indexes = cur.fetchall() + violations = [] - for (table_oid, index_name, table_name, column_names) in unique_indexes: - column_names = ",".join(column_names) - sql = self.get_violated_segments_query(table_name, column_names) - violated_segments = db_connection.query(sql).getresult() - if violated_segments: - violations.append(dict(table_oid=table_oid, - table_name=table_name, - index_name=index_name, - column_names=column_names, - violated_segments=[row[0] for row in violated_segments])) - - return violations + for (table_oid, index_name, table_name, column_names) in unique_indexes: + column_names = ",".join(column_names) + sql = self.get_violated_segments_query(table_name, column_names) + cur.execute(sql) + violated_segments = cur.fetchall() + if violated_segments: + violations.append(dict(table_oid=table_oid, + table_name=table_name, + index_name=index_name, + column_names=column_names, + violated_segments=[row[0] for row in violated_segments])) + return violations def get_violated_segments_query(self, table_name, column_names): return self.violated_segments_query % ( diff --git a/gpMgmt/bin/gpconfig b/gpMgmt/bin/gpconfig index d6e8defbaf3e..44fdfc7ffc2b 100755 --- a/gpMgmt/bin/gpconfig +++ b/gpMgmt/bin/gpconfig @@ -15,7 +15,7 @@ import os import sys import re - +from psycopg2 import DatabaseError try: from gppylib.gpparseopts import OptParser, OptChecker from gppylib.gparray import GpArray @@ -25,7 +25,6 @@ try: from gppylib.commands.gp import * from gppylib.db import dbconn from gppylib.userinput import * - from pg import DatabaseError from gpconfig_modules.segment_guc import SegmentGuc from gpconfig_modules.database_segment_guc import DatabaseSegmentGuc from gpconfig_modules.file_segment_guc import FileSegmentGuc diff --git a/gpMgmt/bin/gpexpand b/gpMgmt/bin/gpexpand index 89c167be16b1..8ad37dad9bce 100755 --- a/gpMgmt/bin/gpexpand +++ b/gpMgmt/bin/gpexpand @@ -17,10 +17,10 @@ import signal import traceback from collections import defaultdict from time import strftime, sleep - +import psycopg2 +from psycopg2 import DatabaseError, OperationalError +from psycopg2 import extras try: - import pg, pgdb - from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.gparray import GpArray, MODE_NOT_SYNC, STATUS_DOWN @@ -32,7 +32,6 @@ try: from gppylib.operations.startSegments import MIRROR_MODE_MIRRORLESS from gppylib.system import configurationInterface, configurationImplGpdb from gppylib.system.environment import GpCoordinatorEnvironment - from pgdb import DatabaseError from gppylib.gpcatalog import COORDINATOR_ONLY_TABLES from gppylib.operations.package import SyncPackages from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup @@ -1880,7 +1879,7 @@ class gpexpand: expansionStopped) dbconn.execSQL(self.conn, sql) self.conn.close() - except pgdb.OperationalError: + except OperationalError: pass except Exception: # schema doesn't exist. Cancel or error during setup @@ -1932,7 +1931,7 @@ class gpexpand: def connect_database(self, dbname): test_url = copy.deepcopy(self.dburl) test_url.pgdb = dbname - c = dbconn.connect(test_url, encoding='UTF8', allowSystemTableMods=True) + c = dbconn.connect(test_url, encoding='UTF8', allowSystemTableMods=True, cursorFactory=psycopg2.extras.NamedTupleCursor) return c def sync_packages(self): @@ -2173,7 +2172,7 @@ class ExpandCommand(SQLCommand): try: status_conn = dbconn.connect(self.status_url, encoding='UTF8') - table_conn = dbconn.connect(self.table_url, encoding='UTF8') + table_conn = dbconn.connect(self.table_url, encoding='UTF8', cursorFactory=psycopg2.extras.NamedTupleCursor) except DatabaseError as ex: if self.options.verbose: logger.exception(ex) diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index 3437dc5ff247..55fe1f78909d 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -35,22 +35,8 @@ sys.exit(2) import platform - -try: - import pg -except ImportError: - try: - from pygresql import pg - except Exception as e: - pass -except Exception as e: - print(repr(e)) - errorMsg = "gpload was unable to import The PyGreSQL Python module (pg.py) - %s\n" % str(e) - sys.stderr.write(str(errorMsg)) - errorMsg = "Please check if you have the correct Visual Studio redistributable package installed.\n" - sys.stderr.write(str(errorMsg)) - sys.exit(2) - +import psycopg2 +from psycopg2 import extras import hashlib import datetime,getpass,os,signal,socket,threading,time,traceback,re import subprocess @@ -562,6 +548,8 @@ def is_keyword(tab): else: return False +def escape_string(string): + return psycopg2.extensions.QuotedString(string).getquoted()[1:-1].decode() def caseInsensitiveDictLookup(key, dictionary): """ diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 44b7632be215..ad28a18862be 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -10,10 +10,8 @@ import os import sys import signal import itertools - +from psycopg2 import DatabaseError try: - import pg - from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.commands.pg import PgControlData @@ -23,7 +21,6 @@ try: from gppylib.db import dbconn from gppylib.userinput import * from gppylib.operations.startSegments import * - from pgdb import DatabaseError from gppylib import gparray, gplog, pgconf, userinput, utils from gppylib.parseutils import line_reader, check_values, canonicalize_address from gppylib.operations.segment_tablespace_locations import get_tablespace_locations diff --git a/gpMgmt/bin/gppylib/commands/base.py b/gpMgmt/bin/gppylib/commands/base.py index c3da5a2aa93a..eba9eb8f4382 100755 --- a/gpMgmt/bin/gppylib/commands/base.py +++ b/gpMgmt/bin/gppylib/commands/base.py @@ -28,7 +28,6 @@ from gppylib import gplog from gppylib import gpsubprocess -from pg import DB logger = gplog.get_default_logger() @@ -628,8 +627,7 @@ def cancel(self): # if self.conn is not set we cannot cancel. if self.cancel_conn: - DB(self.cancel_conn).cancel() - + self.cancel_conn.cancel() def run_remote_commands(name, commands): """ diff --git a/gpMgmt/bin/gppylib/commands/pg.py b/gpMgmt/bin/gppylib/commands/pg.py index 69b629806449..7f2ab9b4e439 100644 --- a/gpMgmt/bin/gppylib/commands/pg.py +++ b/gpMgmt/bin/gppylib/commands/pg.py @@ -14,7 +14,7 @@ from .unix import * from gppylib.commands.base import * from gppylib.commands.gp import RECOVERY_REWIND_APPNAME -from pgdb import DatabaseError +from psycopg2 import DatabaseError logger = get_default_logger() diff --git a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py index 65b49aa424fc..5cfde4b776b5 100644 --- a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py +++ b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py @@ -7,7 +7,7 @@ from mock import call, Mock, patch from gppylib.commands import pg from test.unit.gp_unittest import GpTestCase, run_tests -from pgdb import DatabaseError +from psycopg2 import DatabaseError class TestUnitPgReplicationSlot(GpTestCase): def setUp(self): diff --git a/gpMgmt/bin/gppylib/db/catalog.py b/gpMgmt/bin/gppylib/db/catalog.py index 6214b805c293..80ca66289eb7 100644 --- a/gpMgmt/bin/gppylib/db/catalog.py +++ b/gpMgmt/bin/gppylib/db/catalog.py @@ -6,8 +6,8 @@ """ import copy - -import pg +import os +from contextlib import closing from gppylib import gplog from gppylib.db import dbconn diff --git a/gpMgmt/bin/gppylib/db/dbconn.py b/gpMgmt/bin/gppylib/db/dbconn.py index 1ca0ecf56cb7..24c668d7402b 100644 --- a/gpMgmt/bin/gppylib/db/dbconn.py +++ b/gpMgmt/bin/gppylib/db/dbconn.py @@ -9,9 +9,8 @@ import sys import os import stat - +import psycopg2 try: - import pgdb from gppylib.commands.unix import UserId except ImportError as e: @@ -159,67 +158,44 @@ def canonicalize(s): # 1. pg notice is accessible to a user of connection returned by dbconn.connect(), # lifted from the underlying _pg connection # 2. multiple calls to dbconn.close() should not return an error -class Connection(pgdb.Connection): +class Connection: def __init__(self, connection): - self._notices = collections.deque(maxlen=100) - # we must do an attribute by attribute copy of the notices here - # due to limitations in pg implementation. Wrap with with a - # namedtuple for ease of use. - def handle_notice(notice): - received = {} - for attr in dir(notice): - if attr.startswith('__'): - continue - value = getattr(notice, attr) - received[attr] = value - Notice = collections.namedtuple('Notice', sorted(received)) - self._notices.append(Notice(**received)) - - - self._impl = connection - self._impl._cnx.set_notice_receiver(handle_notice) + self._conn = connection + self._conn.notices = collections.deque(maxlen=100) def __enter__(self): - return self._impl.__enter__() + return self._conn.__enter__() # __exit__() does not close the connection. This is in line with the # python DB API v2 specification (pep-0249), where close() is done on # __del__(), not __exit__(). def __exit__(self, *args): - return self._impl.__exit__(*args) + return self._conn.__exit__(*args) def __getattr__(self, name): - return getattr(self._impl, name) + return getattr(self._conn, name) def notices(self): - notice_list = list(self._notices) - self._notices.clear() + notice_list = list(self._conn.notices) + self._conn.notices.clear() return notice_list # don't return operational error if connection is already closed def close(self): - if not self._impl.closed: - self._impl.close() + if not self._conn.closed: + self._conn.close() def connect(dburl, utility=False, verbose=False, - encoding=None, allowSystemTableMods=False, logConn=True, unsetSearchPath=True): + encoding=None, allowSystemTableMods=False, logConn=True, unsetSearchPath=True, cursorFactory=None): conninfo = { 'user': dburl.pguser, 'password': dburl.pgpass, 'host': dburl.pghost, 'port': dburl.pgport, - # dbname is very subtle, Package pgdb contains a bug it will only escape the string when - # 1. a space in the dbname, and - # 2. there are other keyword arguments of pgdb.connect method - # See issue https://github.com/PyGreSQL/PyGreSQL/issues/77 for details - # The code here is test if there is space, if so, we know pgdb will escape, let's not do here - # if not, let's do escape here since pgdb forget to do. - # - # NB: we always provide port keyword argument to connect method of pgdb, thus - # we will always enter the code path of pgdb.connect of the above escape logic. - 'database': dburl.pgdb if ' ' in dburl.pgdb else dburl.pgdb.replace('\\', '\\\\').replace("'", "\\'"), + 'database': dburl.pgdb, + 'cursor_factory': cursorFactory } # building options @@ -257,22 +233,23 @@ def connect(dburl, utility=False, verbose=False, logFunc = logger.info if dburl.timeout is not None else logger.debug logFunc("Connecting to db {} on host {}".format(dburl.pgdb, dburl.pghost)) - connection = None + conn = None for i in range(retries): try: - connection = pgdb.connect(**conninfo) + conn = psycopg2.connect(**conninfo) + conn.set_session(autocommit=True) break - except pgdb.OperationalError as e: + except psycopg2.OperationalError as e: if 'timeout expired' in str(e): logger.warning('Timeout expired connecting to %s, attempt %d/%d' % (dburl.pgdb, i+1, retries)) continue raise - if connection is None: + if conn is None: raise ConnectionError('Failed to connect to %s' % dburl.pgdb) - return Connection(connection) + return Connection(conn) def execSQL(conn, sql, autocommit=True): """ @@ -286,7 +263,6 @@ def execSQL(conn, sql, autocommit=True): Using `with dbconn.connect() as conn` syntax will override autocommit and complete queries in a transaction followed by a commit on context close """ - conn.autocommit = autocommit with conn.cursor() as cursor: cursor.execute(sql) diff --git a/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py b/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py index 086ccce92981..a03eb4db42b6 100644 --- a/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py +++ b/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py @@ -46,7 +46,7 @@ def test_verbose_mode_allows_warnings_to_be_sent_to_the_client(self): for notice in notices: - if warning in notice.message: + if warning in notice: return # found it! self.fail("Didn't find expected notice '{}' in {!r}".format( diff --git a/gpMgmt/bin/gppylib/gpcatalog.py b/gpMgmt/bin/gppylib/gpcatalog.py index 81b6e5f02e9d..fc2916ac75e4 100644 --- a/gpMgmt/bin/gppylib/gpcatalog.py +++ b/gpMgmt/bin/gppylib/gpcatalog.py @@ -143,7 +143,7 @@ def __init__(self, dbConnection): curs = self._query(version_query) except Exception as e: raise GPCatalogException("Error reading database version: " + str(e)) - self._version = GpVersion(curs.getresult()[0][0]) + self._version = GpVersion(curs.fetchone()[0]) # Read the list of catalog tables from the database try: @@ -153,7 +153,7 @@ def __init__(self, dbConnection): # Construct our internal representation of the catalog - for [oid, relname, relisshared] in curs.getresult(): + for [oid, relname, relisshared] in curs.fetchall(): self._tables[relname] = GPCatalogTable(self, relname) # Note: stupid API returns t/f for boolean value self._tables[relname]._setShared(relisshared == 't') @@ -192,7 +192,9 @@ def _query(self, qry): """ Simple wrapper around querying the database connection """ - return self._dbConnection.query(qry) + cur = self._dbConnection.cursor() + cur.execute(qry) + return cur def _markCoordinatorOnlyTables(self): """ @@ -482,10 +484,10 @@ def __init__(self, parent, name, pkey=None): # exist. raise GPCatalogException("Catalog table %s does not exist" % name) - if cur.ntuples() == 0: + if cur.rowcount == 0: raise GPCatalogException("Catalog table %s does not exist" % name) - for row in cur.getresult(): + for row in cur.fetchall(): (attname, atttype, typname) = row # Mark if the catalog has an oid column @@ -521,7 +523,7 @@ def __init__(self, parent, name, pkey=None): WHERE attrelid = 'pg_catalog.{catname}'::regclass """.format(catname=name) cur = parent._query(qry) - self._pkey = [row[0] for row in cur.getresult()] + self._pkey = [row[0] for row in cur.fetchall()] # Primary key must be in the column list for k in self._pkey: diff --git a/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py b/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py index 49dd622bb628..fac01fd3f513 100644 --- a/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py +++ b/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py @@ -1,9 +1,7 @@ import time - from gppylib.commands import base from gppylib.db import dbconn -import pg - +from contextlib import closing FTS_PROBE_QUERY = 'SELECT pg_catalog.gp_request_fts_probe_scan()' @@ -14,15 +12,19 @@ def __init__(self, logger, worker_pool, timeout): self.timeout = timeout def _trigger_fts_probe(self, dburl): - conn = pg.connect(dbname=dburl.pgdb, - host=dburl.pghost, - port=dburl.pgport, - opt=None, - user=dburl.pguser, - passwd=dburl.pgpass, - ) - conn.query(FTS_PROBE_QUERY) - conn.close() + start_time = time.time() + while True: + try: + with closing(dbconn.connect(dburl)) as conn: + with conn.cursor() as cur: + cur.execute(FTS_PROBE_QUERY) + break + except Exception as e: + now = time.time() + if now < start_time + self.timeout: + continue + else: + raise RuntimeError("FTS probing did not complete in {} seconds.".format(self.timeout)) def reconfigure(self): # issue a distributed query to make sure we pick up the fault @@ -36,15 +38,15 @@ def reconfigure(self): # Empty block of 'BEGIN' and 'END' won't start a distributed transaction, # execute a DDL query to start a distributed transaction. # so the primaries'd better be up - conn = dbconn.connect(dburl) - conn.cursor().execute('CREATE TEMP TABLE temp_test(a int)') - conn.cursor().execute('COMMIT') + with closing(dbconn.connect(dburl)) as conn: + with conn.cursor() as cur: + cur.execute('BEGIN') + cur.execute('CREATE TEMP TABLE temp_test(a int)') + cur.execute('COMMIT') + break except Exception as e: now = time.time() if now < start_time + self.timeout: continue else: raise RuntimeError("Mirror promotion did not complete in {0} seconds.".format(self.timeout)) - else: - conn.close() - break diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py index 9d86071bafd7..d4c2a0c26e72 100644 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py @@ -4,8 +4,7 @@ from gppylib.operations.segment_reconfigurer import SegmentReconfigurer, FTS_PROBE_QUERY from gppylib.test.unit.gp_unittest import GpTestCase -import pg -import pgdb +import psycopg2 import mock from mock import Mock, patch, call, MagicMock import contextlib @@ -38,22 +37,22 @@ def setUp(self): self.apply_patches([ patch('gppylib.db.dbconn.connect', new=self.connect), patch('gppylib.db.dbconn.DbURL', return_value=self.db_url), - patch('pg.connect'), + patch('psycopg2.connect'), ]) def test_it_triggers_fts_probe(self): reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=self.worker_pool, timeout=self.timeout) reconfigurer.reconfigure() - pg.connect.assert_has_calls([ - call(dbname=self.db, host=self.host, port=self.port, opt=None, user=self.user, passwd=self.passwd), + psycopg2.connect.assert_has_calls([ + call(dbname=self.db, host=self.host, port=self.port, options=None, user=self.user, password=self.passwd), call().query(FTS_PROBE_QUERY), call().close(), ] ) def test_it_retries_the_connection(self): - self.connect.configure_mock(side_effect=[pgdb.DatabaseError, pgdb.DatabaseError, self.conn]) + self.connect.configure_mock(side_effect=[psycopg2.DatabaseError, psycopg2.DatabaseError, self.conn]) reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=self.worker_pool, timeout=self.timeout) @@ -74,7 +73,7 @@ def fail_for_five_minutes(): # leap forward 300 seconds new_time += self.timeout / 2 now_mock.configure_mock(return_value=new_time) - yield pgdb.DatabaseError + yield psycopg2.DatabaseError self.connect.configure_mock(side_effect=fail_for_five_minutes()) @@ -87,3 +86,27 @@ def fail_for_five_minutes(): self.connect.assert_has_calls([call(self.db_url), call(self.db_url), ]) self.conn.close.assert_has_calls([]) + + @patch('time.time') + def test_it_gives_up_after_600_seconds_2(self, now_mock): + start_datetime = datetime.datetime(2023, 7, 27, 16, 0, 0) + start_time = time.mktime(start_datetime.timetuple()) + now_mock.configure_mock(return_value=start_time) + + def fail_for_ten_minutes(): + new_time = start_time + # leap forward 600 seconds + new_time += self.timeout + now_mock.configure_mock(return_value=new_time) + yield psycopg2.DatabaseError + + self.connect.configure_mock(side_effect=fail_for_ten_minutes()) + + reconfigurer = SegmentReconfigurer(logger=self.logger, + worker_pool=self.worker_pool, timeout=self.timeout) + with self.assertRaises(RuntimeError) as context: + reconfigurer.reconfigure() + self.assertEqual("FTS probing did not complete in {} seconds.".format(self.timeout), context.exception.message) + + self.connect.assert_has_calls([call(self.db_url)]) + self.conn.close.assert_has_calls([]) diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py index 38fcd327dee4..e62aa7a40b72 100755 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py @@ -11,7 +11,7 @@ from gppylib.operations.test_utils_helper import TestOperation, RaiseOperation, RaiseOperation_Unpicklable, RaiseOperation_Safe, ExceptionWithArgs from operations.unix import ListFiles from test.unit.gp_unittest import GpTestCase, run_tests -from pg import DatabaseError +from psycopg2 import DatabaseError class UtilsTestCase(GpTestCase): """ diff --git a/gpMgmt/bin/gppylib/operations/test_utils_helper.py b/gpMgmt/bin/gppylib/operations/test_utils_helper.py index 109bc83bb878..74cfeda2bc01 100755 --- a/gpMgmt/bin/gppylib/operations/test_utils_helper.py +++ b/gpMgmt/bin/gppylib/operations/test_utils_helper.py @@ -1,4 +1,5 @@ from gppylib.operations import Operation +import psycopg2 """ These objects needed for gppylib.operations.test.test_utils are pulled out of said file for @@ -37,5 +38,4 @@ def __init__(self, x, y): class RaiseOperation_Unpicklable(Operation): def execute(self): - import pg - raise pg.DatabaseError() + raise psycopg2.DatabaseError() diff --git a/gpMgmt/bin/gppylib/programs/clsSystemState.py b/gpMgmt/bin/gppylib/programs/clsSystemState.py index 5c120a592a47..62aba744904f 100644 --- a/gpMgmt/bin/gppylib/programs/clsSystemState.py +++ b/gpMgmt/bin/gppylib/programs/clsSystemState.py @@ -10,7 +10,7 @@ import sys, os import re import collections -import pgdb +import psycopg2 from contextlib import closing from gppylib import gparray, gplog from gppylib.commands import base, gp @@ -1026,7 +1026,7 @@ def _get_unsync_segs_add_wal_remaining_bytes(data, gpArray): wal_sync_bytes_out = 'Unknown' unsync_segs.append(s) data.addValue(VALUE__REPL_SYNC_REMAINING_BYTES, wal_sync_bytes_out) - except pgdb.InternalError: + except (psycopg2.InternalError, psycopg2.OperationalError): logger.warning('could not query segment {} ({}:{})'.format( s.dbid, s.hostname, s.port )) @@ -1098,7 +1098,7 @@ def _add_replication_info(data, primary, mirror): cursor.close() - except pgdb.InternalError: + except (psycopg2.InternalError, psycopg2.OperationalError): logger.warning('could not query segment {} ({}:{})'.format( primary.dbid, primary.hostname, primary.port )) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py b/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py index 9888472bb436..b9e691a7e567 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py @@ -11,7 +11,7 @@ class GpCheckCatTestCase(GpTestCase): def setUp(self): self.logger = Mock(spec=['log', 'info', 'debug', 'error']) - self.db_connection = Mock(spec=['close', 'query']) + self.db_connection = Mock(spec=['close', 'cursor']) self.autoCast = {'regproc': '::oid', 'regprocedure': '::oid', 'regoper': '::oid', @@ -25,9 +25,10 @@ def setUp(self): self.full_join_cat_tables = set(['pg_attribute','gp_distribution_policy','pg_appendonly','pg_constraint','pg_index']) self.foreign_key_check= Mock(spec=['runCheck']) self.foreign_key_check.runCheck.return_value = [] - self.db_connection.query.return_value.ntuples.return_value = 2 - self.db_connection.query.return_value.listfields.return_value = ['pkey1', 'pkey2'] - self.db_connection.query.return_value.getresult.return_value = [('r1','r2'), ('r3','r4')] + + self.db_connection.cursor.return_value.rowcount = 2 + self.db_connection.cursor.return_value.description = [('pkey1',), ('pkey2',)] + self.db_connection.cursor.return_value.fetchall.return_value = [('r1','r2'), ('r3','r4')] def test_get_fk_query_left_join_returns_the_correct_query(self): @@ -127,7 +128,7 @@ def test_checkTableForeignKey__returns_correct_join_query(self, log_literal_mock self.assertEqual(len(issue_list) , 2) self.assertEqual(issue_list[0], ('pg_class', ['pkey1', 'pkey2'], [('r1', 'r2'), ('r3', 'r4')])) self.assertEqual(issue_list[1], ('arbitrary_catalog_table', ['pkey1', 'pkey2'], [('r1', 'r2'), ('r3', 'r4')])) - self.assertEqual(self.db_connection.query.call_count, 2) + self.assertEqual(self.db_connection.cursor.call_count, 2) def __generate_pg_class_call(table, primary_key_cat_name, col_type, with_filter=True): if with_filter: @@ -168,7 +169,7 @@ def __generate_pg_class_call(table, primary_key_cat_name, col_type, with_filter= self.assertEqual(fk_query_full_join_mock.call_count, 0) fk_query_left_join_mock.assert_has_calls(foreign_key_mock_calls_left, any_order=False) - self.db_connection.query.call_count = 0 + self.db_connection.cursor.call_count = 0 fk_query_full_join_mock.call_count = 0 fk_query_left_join_mock.call_count = 0 diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py index 63f2d88fe89f..3b476a3a0bb7 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py @@ -18,11 +18,11 @@ def setUp(self): self.subject = imp.load_source('gpcheckcat', gpcheckcat_file) self.subject.check_gpexpand = lambda : (True, "") - self.db_connection = Mock(spec=['close', 'query']) + self.db_connection = Mock(spec=['close', 'cursor', 'set_session']) self.unique_index_violation_check = Mock(spec=['runCheck']) self.foreign_key_check = Mock(spec=['runCheck', 'checkTableForeignKey']) self.apply_patches([ - patch("gpcheckcat.pg.connect", return_value=self.db_connection), + patch("gpcheckcat.connect", return_value=self.db_connection), patch("gpcheckcat.UniqueIndexViolationCheck", return_value=self.unique_index_violation_check), patch("gpcheckcat.ForeignKeyCheck", return_value=self.foreign_key_check), patch('os.environ', new={}), @@ -129,23 +129,26 @@ def test_drop_leaked_schemas__when_leaked_schemas_exist__reports_which_schemas_a self.assertIn(expected_message, log_messages) def test_automatic_thread_count(self): - self.db_connection.query.return_value.getresult.return_value = [[0]] + self.db_connection.cursor.return_value.fetchall.return_value = [[0]] self._run_batch_size_experiment(100) self._run_batch_size_experiment(101) + @patch('gpcheckcat.getversion', return_value='4.3') @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @patch('gppylib.gplog.log_literal') - def test_truncate_batch_size(self, mock_log, mock_gpcheckcat, mock_sys_exit): + def test_truncate_batch_size(self, mock_log, mock_sys_exit, mock_gpcatalog, mock_version): self.subject.GV.opt['-B'] = 300 # override the setting from available memory # setup conditions for 50 primaries and plenty of RAM such that max threads > 50 primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.getresult.return_value = [['4.3']] - self.db_connection.query.return_value.dictresult.return_value = primaries + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries testargs = ['some_string','-port 1', '-R foo'] @@ -221,10 +224,11 @@ def test_checkForeignKey__no_arg(self, process_foreign_key_mock): self.foreign_key_check.runCheck.assert_called_once_with(cat_tables) # Test gpcheckat -C option with checkForeignKey + @patch('gpcheckcat.getversion', return_value='4.3') @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @patch('gpcheckcat.checkTableMissingEntry') - def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3): + def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3, mock4): self.subject.checkForeignKey = Mock() gpcat_class_mock = Mock(spec=['getCatalogTable']) cat_obj_mock = Mock() @@ -234,8 +238,12 @@ def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3): for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.getresult.return_value = [['4.3']] - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries self.subject.GV.opt['-C'] = 'pg_class' @@ -314,7 +322,13 @@ def test_skip_one_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s test2'] @@ -330,7 +344,13 @@ def test_skip_multiple_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s', "test1, test2"] @@ -346,7 +366,11 @@ def test_skip_test_warning(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s', "test_invalid, test2"] @@ -365,7 +389,13 @@ def test_run_multiple_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-R', "test1, test2"] diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py index 298cf80d61c1..a3e3bd5c2026 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py @@ -5,11 +5,10 @@ import shutil import sys import tempfile - +from psycopg2 import DatabaseError from gppylib.gparray import Segment, GpArray, SegmentPair from gpconfig_modules.parse_guc_metadata import ParseGuc import errno -from pg import DatabaseError from .gp_unittest import * from unittest.mock import * diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py index e9c60760aebb..4858e80e65a4 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py @@ -1,6 +1,6 @@ import unittest import mock -import pgdb +import psycopg2 import tempfile from gppylib import gparray @@ -228,7 +228,7 @@ def test_add_replication_info_adds_unknowns_if_primary_is_down(self): @mock.patch('gppylib.db.dbconn.connect', autospec=True) def test_add_replication_info_adds_unknowns_if_connection_cannot_be_made(self, mock_connect): # Simulate a connection failure in dbconn.connect(). - mock_connect.side_effect = pgdb.InternalError('connection failure forced by unit test') + mock_connect.side_effect = psycopg2.InternalError('connection failure forced by unit test') GpSystemStateProgram._add_replication_info(self.data, self.primary, self.mirror) self.assertEqual('Unknown', self.data.getStrValue(self.mirror, VALUE__REPL_SENT_LSN)) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py b/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py index 1eda2e72a250..1cd81684c235 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py @@ -6,45 +6,57 @@ class LeakedSchemaDropperTestCase(GpTestCase): def setUp(self): - self.db_connection = Mock(spec=['query']) - - two_leaked_schemas = Mock() - two_leaked_schemas.getresult.return_value = [ + self.db_connection = Mock(spec=['cursor']) + self.db_connection.cursor.return_value.fetchall.return_value = [ ('fake_leak_1', 'something_else'), ('some"test"special_#;character--schema', 'something_else') ] - self.db_connection.query.return_value = two_leaked_schemas - self.subject = LeakedSchemaDropper() def test_drop_leaked_schemas__returns_a_list_of_leaked_schemas(self): + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [ + ('fake_leak_1', 'something_else'), + ('some"test"special_#;character--schema', 'something_else') + ] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), ['fake_leak_1', 'some"test"special_#;character--schema']) def test_drop_leaked_schemas__when_there_are_no_leaked_schemas__returns_an_empty_list(self): - no_leaked_schemas = Mock() - no_leaked_schemas.getresult.return_value = [] - self.db_connection.query.return_value = no_leaked_schemas - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__when_query_returns_null_schema__returns_an_empty_list(self): - null_leaked_schema = Mock() - null_leaked_schema.getresult.return_value = [(None, 'something_else')] - self.db_connection.query.return_value = null_leaked_schema - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [(None, 'something_else')] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__when_query_returns_null__returns_an_empty_list(self): - self.db_connection.query.return_value = None - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__drops_orphaned_and_leaked_schemas(self): + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [ + ('fake_leak_1', 'something_else'), + ('some"test"special_#;character--schema', 'something_else') + ] self.subject.drop_leaked_schemas(self.db_connection) drop_query_expected_list = [call("DROP SCHEMA IF EXISTS \"fake_leak_1\" CASCADE;"), call("DROP SCHEMA IF EXISTS \"some\"\"test\"\"special_#;character--schema\" CASCADE;")] - self.db_connection.query.assert_has_calls(drop_query_expected_list) + self.db_connection.cursor.return_value.__enter__.return_value.execute.assert_has_calls(drop_query_expected_list) if __name__ == '__main__': diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py b/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py index c9306d69b97c..19ddb7732324 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py @@ -9,33 +9,37 @@ def setUp(self): self.subject = UniqueIndexViolationCheck() self.index_query_result = Mock() - self.index_query_result.getresult.return_value = [ + self.index_query_result.fetchall.return_value = [ (9001, 'index1', 'table1', ['index1_column1','index1_column2']), (9001, 'index2', 'table1', ['index2_column1','index2_column2']) ] - self.violated_segments_query_result = Mock() - - self.db_connection = Mock(spec=['query']) - self.db_connection.query.side_effect = self.mock_query_return_value - - def mock_query_return_value(self, query_string): - if query_string == UniqueIndexViolationCheck.unique_indexes_query: - return self.index_query_result - else: - return self.violated_segments_query_result + self.db_connection = Mock(spec=['cursor']) + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) def test_run_check__when_there_are_no_issues(self): - self.violated_segments_query_result.getresult.return_value = [] + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.side_effect = [ + [ + (9001, 'index1', 'table1', ['index1_column1','index1_column2']), + (9001, 'index2', 'table1', ['index2_column1','index2_column2']) + ], + [], + [], + ] violations = self.subject.runCheck(self.db_connection) self.assertEqual(len(violations), 0) def test_run_check__when_index_is_violated(self): - self.violated_segments_query_result.getresult.side_effect = [ + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.side_effect = [ + [ + (9001, 'index1', 'table1', ['index1_column1','index1_column2']), + (9001, 'index2', 'table1', ['index2_column1','index2_column2']) + ], [(-1,), (0,), (1,)], - [(-1,)] + [(-1,)], ] violations = self.subject.runCheck(self.db_connection) diff --git a/gpMgmt/bin/gppylib/utils.py b/gpMgmt/bin/gppylib/utils.py index fead818d3ac1..42c77e620ab3 100644 --- a/gpMgmt/bin/gppylib/utils.py +++ b/gpMgmt/bin/gppylib/utils.py @@ -5,8 +5,7 @@ from sys import * from xml.dom import minidom from xml.dom import Node - -import pgdb +import psycopg2 from gppylib.gplog import * logger = get_default_logger() @@ -503,14 +502,12 @@ def escapeDoubleQuoteInSQLString(string, forceDoubleQuote=True): string = '"' + string + '"' return string - -def Escape(query_str): - return pgdb.escape_string(query_str) - +def escape_string(string): + return psycopg2.extensions.QuotedString(string).getquoted()[1:-1].decode() def escapeArrayElement(query_str): # also escape backslashes and double quotes, in addition to the doubling of single quotes - return pgdb.escape_string(query_str.encode(errors='backslashreplace')).decode(errors='backslashreplace').replace('\\','\\\\').replace('"','\\"') + return escape_string(query_str.encode(errors='backslashreplace')).encode().decode(errors='backslashreplace').replace('\\','\\\\').replace('"','\\"') # Transform Python list to Postgres array literal (of the form: '{...}') @@ -593,7 +590,7 @@ def formatInsertValuesList(row, starelid, inclHLL): # Format stavalues5 for an hll slot elif i == 30 and hll: if inclHLL: - val = '\'{"%s"}\'' % pgdb.escape_bytea(val[0]) + val = '\'{\\%s}\'' % val[0] rowVals.append('\t{0}::{1}'.format(val, 'bytea[]')) else: rowVals.append('\t{0}'.format('NULL::int4[]')) diff --git a/gpMgmt/bin/gpsd b/gpMgmt/bin/gpsd index daa8146ab913..363ed9d46ed8 100755 --- a/gpMgmt/bin/gpsd +++ b/gpMgmt/bin/gpsd @@ -12,8 +12,8 @@ import re from contextlib import closing from distutils.version import LooseVersion from optparse import OptionParser -import pgdb -from gppylib.utils import formatInsertValuesList, Escape +from gppylib.utils import formatInsertValuesList, escape_string +import psycopg2 gpsd_version = '%prog 1.0' @@ -43,7 +43,7 @@ def get_num_segments(cursor): query = "select count(*) from gp_segment_configuration where role='p' and content >=0;" try: cursor.execute(query) - except pgdb.DatabaseError as e: + except psycopg2.DatabaseError as e: sys.stderr.write('\nError while trying to retrieve number of segments.\n\n' + str(e) + '\n\n') sys.exit(1) vals = cursor.fetchone() @@ -83,7 +83,13 @@ def dumpTupleCount(cur): def dumpStats(cur, inclHLL): - query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, pgs.* ' \ + query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, ' \ + 'pgs.starelid, pgs.staattnum, pgs.stainherit, pgs.stanullfrac, pgs.stawidth, pgs.stadistinct, ' \ + 'pgs.stakind1, pgs.stakind2, pgs.stakind3, pgs.stakind4, pgs.stakind5, ' \ + 'pgs.staop1, pgs.staop2, pgs.staop3, pgs.staop4, pgs.staop5, ' \ + 'pgs.stacoll1, pgs.stacoll2, pgs.stacoll3, pgs.stacoll4, pgs.stacoll5, ' \ + 'pgs.stanumbers1, pgs.stanumbers2, pgs.stanumbers3, pgs.stanumbers4, pgs.stanumbers5, ' \ + 'pgs.stavalues1::text::text[], pgs.stavalues2::text::text[], pgs.stavalues3::text::text[], pgs.stavalues4::text::text[], pgs.stavalues5::text::text[] ' \ 'FROM pg_class pgc, pg_statistic pgs, pg_namespace pgn, pg_attribute pga, pg_type pgt, pg_namespace pgtn ' \ 'WHERE pgc.relnamespace = pgn.oid and pgn.nspname NOT IN ' + \ sysnslist + \ @@ -102,7 +108,7 @@ def dumpStats(cur, inclHLL): cur.execute(query) for vals in ResultIter(cur): - starelid = "'%s.%s'::regclass" % (Escape(vals[1]), Escape(vals[0])) + starelid = "'%s.%s'::regclass" % (escape_string(vals[1]), escape_string(vals[0])) rowVals = formatInsertValuesList(vals, starelid, inclHLL) print(pstring.format(vals[0], vals[2], ',\n'.join(rowVals))) @@ -153,7 +159,7 @@ def main(): 'options': pgoptions } num_segments = 0 - with closing(pgdb.connect(**connectionInfo)) as connection: + with closing(psycopg2.connect(**connectionInfo)) as connection: with closing(connection.cursor()) as cursor: num_segments = get_num_segments(cursor) sys.stdout.writelines(['\n-- Greenplum database Statistics Dump', @@ -188,7 +194,7 @@ def main(): sys.stdout.flush() try: - with closing(pgdb.connect(**connectionInfo)) as connection: + with closing(psycopg2.connect(**connectionInfo)) as connection: with closing(connection.cursor()) as cursor: dumpTupleCount(cursor) dumpStats(cursor, inclHLL) @@ -197,11 +203,11 @@ def main(): 'which requires some data elements to be included in the output file.\n', 'Please review output file to ensure it is within corporate policy to transport the output file.\n']) - except pgdb.DatabaseError as err: # catch *all* exceptions + except psycopg2.DatabaseError as err: # catch *all* exceptions sys.stderr.write('Error while dumping statistics:\n') sys.stderr.write(err.message) sys.exit(1) if __name__ == "__main__": - main() + main() diff --git a/gpMgmt/bin/minirepro b/gpMgmt/bin/minirepro index 684be762d5c9..682a659ca9ea 100755 --- a/gpMgmt/bin/minirepro +++ b/gpMgmt/bin/minirepro @@ -61,10 +61,10 @@ minirepro gptest -h locahost -U gpadmin -p 4444 -q ~/in.sql -f ~/out.sql import pwd import os, sys, re, json, platform, subprocess -import pgdb +import psycopg2 from optparse import OptionParser from datetime import datetime -from gppylib.utils import formatInsertValuesList, Escape +from gppylib.utils import formatInsertValuesList, escape_string version = '1.13' PATH_PREFIX = '/tmp/' @@ -97,7 +97,7 @@ def get_server_version(cursor): query = "select version()" try: cursor.execute(query) - except pgdb.DatabaseError as e: + except psycopg2.DatabaseError as e: sys.stderr.write('\nError while trying to find GPDB version.\n\n' + str(e) + '\n\n') sys.exit(1) vals = cursor.fetchone() @@ -107,7 +107,7 @@ def get_num_segments(cursor): query = "select count(*) from gp_segment_configuration where role='p' and content >=0;" try: cursor.execute(query) - except pgdb.DatabaseError as e: + except psycopg2.DatabaseError as e: sys.stderr.write('\nError while trying to retrieve number of segments.\n\n' + str(e) + '\n\n') sys.exit(1) vals = cursor.fetchone() @@ -136,7 +136,7 @@ def dump_query(connectionInfo, query_file): with open(query_file, 'r') as query_f: sql_text = query_f.read() - query = "select pg_catalog.gp_dump_query_oids('%s')" % Escape(sql_text) + query = "select pg_catalog.gp_dump_query_oids('%s')" % escape_string(sql_text) toolkit_sql = PATH_PREFIX + 'toolkit.sql' with open(toolkit_sql, 'w') as toolkit_f: @@ -187,7 +187,7 @@ def pg_dump_object(mr_query, connectionInfo, envOpts): out_file = PATH_PREFIX + PGDUMP_FILE dmp_cmd = 'pg_dump -h %s -p %s -U %s -sxO %s' % connectionInfo dmp_cmd = "%s --relation-oids %s --function-oids %s -f %s" % \ - (dmp_cmd, mr_query.relids, mr_query.funcids, Escape(out_file)) + (dmp_cmd, mr_query.relids, mr_query.funcids, escape_string(out_file)) print(dmp_cmd) p = subprocess.Popen(dmp_cmd, shell=True, stderr=subprocess.PIPE, env=envOpts) _, errormsg = p.communicate() @@ -213,11 +213,17 @@ def dump_tuple_count(cur, oid_str, f_out): for col, val, typ in zip(columns[2:], vals[2:], types): # i.e. relpages = 1::int, reltuples = 1.0::real lines.append('\t%s = %s::%s' % (col, val, typ)) - updateStmt = templateStmt.format(Escape(',\n'.join(lines)), Escape(vals[0]), Escape(vals[1])) + updateStmt = templateStmt.format(escape_string(',\n'.join(lines)), escape_string(vals[0]), escape_string(vals[1])) f_out.writelines(updateStmt) def dump_stats(cur, oid_str, f_out, inclHLL): - query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, pgs.* ' \ + query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, ' \ + 'pgs.starelid, pgs.staattnum, pgs.stainherit, pgs.stanullfrac, pgs.stawidth, pgs.stadistinct, ' \ + 'pgs.stakind1, pgs.stakind2, pgs.stakind3, pgs.stakind4, pgs.stakind5, ' \ + 'pgs.staop1, pgs.staop2, pgs.staop3, pgs.staop4, pgs.staop5, ' \ + 'pgs.stacoll1, pgs.stacoll2, pgs.stacoll3, pgs.stacoll4, pgs.stacoll5, ' \ + 'pgs.stanumbers1, pgs.stanumbers2, pgs.stanumbers3, pgs.stanumbers4, pgs.stanumbers5, ' \ + 'pgs.stavalues1::text::text[], pgs.stavalues2::text::text[], pgs.stavalues3::text::text[], pgs.stavalues4::text::text[], pgs.stavalues5::text::text[] ' \ 'FROM pg_class pgc, pg_statistic pgs, pg_namespace pgn, pg_attribute pga, pg_type pgt, pg_namespace pgtn ' \ 'WHERE pgc.relnamespace = pgn.oid and pgc.oid in (%s) ' \ 'and pgn.nspname NOT LIKE \'pg_temp_%%\' ' \ @@ -239,7 +245,7 @@ def dump_stats(cur, oid_str, f_out, inclHLL): for vals in result_iter(cur): schemaname = vals[1] - starelid = "'%s.%s'::regclass" % (Escape(vals[1]), Escape(vals[0])) + starelid = "'%s.%s'::regclass" % (escape_string(vals[1]), escape_string(vals[0])) rowVals = formatInsertValuesList(vals, starelid, inclHLL) # For non-catalog tables we don't need to delete stats first @@ -248,7 +254,7 @@ def dump_stats(cur, oid_str, f_out, inclHLL): if schemaname != 'pg_catalog': linecomment = '-- ' # This will comment out the DELETE query - f_out.writelines(pstring.format(Escape(vals[0]), Escape(vals[2]), linecomment, starelid, vals[6], ',\n'.join(rowVals))) + f_out.writelines(pstring.format(escape_string(vals[0]), escape_string(vals[2]), linecomment, starelid, vals[6], ',\n'.join(rowVals))) def main(): parser = parse_cmd_line() @@ -299,7 +305,7 @@ def main(): } print("Connecting to database: host=%s, port=%s, user=%s, db=%s ..." % connectionInfo) - conn = pgdb.connect(**connectionDict) + conn = psycopg2.connect(**connectionDict) cursor = conn.cursor() # get server version, which is dumped to minirepro output file @@ -346,7 +352,7 @@ def main(): # first create schema DDLs print("Writing schema DDLs ...") - table_schemas = ["CREATE SCHEMA %s;\n" % Escape(schema) for schema in mr_query.schemas if schema != 'public'] + table_schemas = ["CREATE SCHEMA %s;\n" % escape_string(schema) for schema in mr_query.schemas if schema != 'public'] f_out.writelines(table_schemas) # write relation and function DDLs @@ -392,4 +398,4 @@ def main(): print('Please review output file to ensure it is within corporate policy to transport the output file.') if __name__ == "__main__": - main() + main() diff --git a/gpMgmt/sbin/gpsegstop.py b/gpMgmt/sbin/gpsegstop.py index 188cd2f3c1f7..37c0604aba5a 100755 --- a/gpMgmt/sbin/gpsegstop.py +++ b/gpMgmt/sbin/gpsegstop.py @@ -23,7 +23,6 @@ from gppylib.commands import gp from gppylib.commands.gp import SEGMENT_STOP_TIMEOUT_DEFAULT, DEFAULT_SEGHOST_NUM_WORKERS from gppylib.commands import pg -from gppylib.db import dbconn from gppylib import pgconf from gppylib.commands.gp import is_pid_postmaster diff --git a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py index 7d1aa3481ade..ce54dbb2c4bf 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/mgmt_utils.py @@ -19,7 +19,8 @@ from datetime import datetime, timedelta from os import path from contextlib import closing - +import psycopg2 +from psycopg2 import extras from gppylib.gparray import GpArray, ROLE_PRIMARY, ROLE_MIRROR from gppylib.commands.gp import SegmentStart, GpStandbyStart, CoordinatorStop from gppylib.commands import gp @@ -3077,31 +3078,41 @@ def impl(context, table_name): dbname = 'gptest' conn = dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False) context.long_run_select_only_conn = conn + cursor = conn.cursor() + context.long_run_select_only_cursor = cursor + + # Start a readonly transaction. + cursor.execute("BEGIN") query = """SELECT gp_segment_id, * from %s order by 1, 2""" % table_name - data_result = dbconn.query(conn, query).fetchall() + cursor.execute(query) + data_result = cursor.fetchall() + context.long_run_select_only_data_result = data_result query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] context.long_run_select_only_xid = xid @then('verify that long-run read-only transaction still exists on {table_name}') def impl(context, table_name): dbname = 'gptest' - conn = context.long_run_select_only_conn + cursor = context.long_run_select_only_cursor query = """SELECT gp_segment_id, * from %s order by 1, 2""" % table_name - data_result = dbconn.query(conn, query).fetchall() + cursor.execute(query) + data_result = cursor.fetchall() query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] if (xid != context.long_run_select_only_xid or data_result != context.long_run_select_only_data_result): error_str = "Incorrect xid or select result of long run read-only transaction: \ - xid(before %s, after %), result(before %s, after %s)" - raise Exception(error_str % (context.long_run_select_only_xid, xid, context.long_run_select_only_data_result, data_result)) + xid(before {}, after {}), result(before {}, after {})" + raise Exception(error_str.format(context.long_run_select_only_xid, xid, context.long_run_select_only_data_result, data_result)) @given('a long-run transaction starts') def impl(context): @@ -3109,30 +3120,36 @@ def impl(context): conn = dbconn.connect(dbconn.DbURL(dbname=dbname), unsetSearchPath=False) context.long_run_conn = conn + cursor = conn.cursor() + context.long_run_cursor = cursor + + cursor.execute("BEGIN") + query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] context.long_run_xid = xid @then('verify that long-run transaction aborted for changing the catalog by creating table {table_name}') def impl(context, table_name): - dbname = 'gptest' - conn = context.long_run_conn + cursor = context.long_run_cursor query = """SELECT txid_current()""" - xid = dbconn.querySingleton(conn, query) + cursor.execute(query) + xid = cursor.fetchone()[0] if context.long_run_xid != xid: raise Exception("Incorrect xid of long run transaction: before %s, after %s" % (context.long_run_xid, xid)); query = """CREATE TABLE %s (a INT)""" % table_name try: - data_result = dbconn.query(conn, query) - except Exception as msg: - key_msg = "FATAL: cluster is expanded" - if key_msg not in msg.__str__(): - raise Exception("transaction not abort correctly, errmsg:%s" % msg) + cursor.execute(query) + except Exception as e: + key_msg = "cluster is expanded from" + if key_msg not in str(e): + raise Exception("transaction not abort correctly, errmsg:%s" % str(e)) else: - raise Exception("transaction not abort, result:%s" % data_result) + raise Exception("transaction not abort") @when('verify that the cluster has {num_of_segments} new segments') @then('verify that the cluster has {num_of_segments} new segments') @@ -3735,7 +3752,7 @@ def impl(context): @then('the database locales are saved') def impl(context): - with closing(dbconn.connect(dbconn.DbURL())) as conn: + with closing(dbconn.connect(dbconn.DbURL(), cursorFactory=psycopg2.extras.NamedTupleCursor)) as conn: rows = dbconn.query(conn, "SELECT name, setting FROM pg_settings WHERE name LIKE 'lc_%'").fetchall() context.database_locales = {row.name: row.setting for row in rows} diff --git a/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py b/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py index aa9b4a011c1c..1c20e690d534 100644 --- a/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py +++ b/gpMgmt/test/behave/mgmt_utils/steps/replication_slots_utils.py @@ -28,7 +28,7 @@ def create_cluster(context, with_mirrors=True): cd ../gpAux/gpdemo; \ export DEMO_PORT_BASE={port_base} && \ export NUM_PRIMARY_MIRROR_PAIRS={num_primary_mirror_pairs} && \ - export WITH_MIRRORS={with_mirrors} && \A + export WITH_MIRRORS={with_mirrors} && \ ./demo_cluster.sh -d && ./demo_cluster.sh -c && \ ./demo_cluster.sh """.format(port_base=os.getenv('PORT_BASE', 15432), @@ -108,18 +108,17 @@ def step_impl(context): def step_impl(context): result_cursor = query_sql( "postgres", - "select pg_get_replication_slots() from gp_dist_random('gp_id') order by gp_segment_id" + "select (pg_get_replication_slots()).* from gp_dist_random('gp_id') order by gp_segment_id" ) if result_cursor.rowcount != context.current_cluster_size: raise Exception("expected all %d primaries to have replication slots, only %d have slots" % (context.current_cluster_size, results.rowcount)) - for content_id, result in enumerate(result_cursor.fetchall()): - pg_rep_slot = result[0] - if (pg_rep_slot[0], pg_rep_slot[2], pg_rep_slot[4]) != ('internal_wal_replication_slot','physical','f') : + for content_id, pg_rep_slot in enumerate(result_cursor.fetchall()): + if (pg_rep_slot[0], pg_rep_slot[2], pg_rep_slot[4]) != ('internal_wal_replication_slot', 'physical', False) : raise Exception( "expected replication slot to be active for content id %d, got %s" % - (content_id, result[0]) + (content_id, pg_rep_slot) ) @then('the mirrors should not have replication slots') diff --git a/gpMgmt/test/behave_utils/gpexpand_dml.py b/gpMgmt/test/behave_utils/gpexpand_dml.py index 8658c7e58ff5..55f0e37b34dd 100755 --- a/gpMgmt/test/behave_utils/gpexpand_dml.py +++ b/gpMgmt/test/behave_utils/gpexpand_dml.py @@ -31,9 +31,15 @@ def __init__(self, dbname, dmltype): def run(self): conn = dbconn.connect(dbconn.DbURL(dbname=self.dbname), unsetSearchPath=False) + with conn.cursor() as cur: + cur.execute("BEGIN") + self.loop(conn) self.verify(conn) + with conn.cursor() as cur: + cur.execute("COMMIT") + conn.commit() conn.close() @@ -109,13 +115,15 @@ def loop_step(self): def verify(self, conn): sql = ''' select c1 from {tablename} order by c1; - '''.format(tablename=self.tablename, counter=self.counter) - results = dbconn.query(conn, sql).fetchall() + '''.format(tablename=self.tablename) + with conn.cursor() as cur: + cur.execute(sql) + results = cur.fetchall() - for i in range(0, self.counter): - if i != int(results[i][0]): - self.report_incorrect_result() - return + for i in range(self.counter): + if i != int(results[i][0]): + self.report_incorrect_result() + return class TestUpdate(TestDML): datasize = 1000 @@ -135,13 +143,15 @@ def loop_step(self): def verify(self, conn): sql = ''' select c2 from {tablename} order by c1; - '''.format(tablename=self.tablename, counter=self.counter) - results = dbconn.query(conn, sql).fetchall() + '''.format(tablename=self.tablename) + with conn.cursor() as cur: + cur.execute(sql) + results = cur.fetchall() - for i in range(0, self.datasize): - if i + self.counter - 1 != int(results[i][0]): - self.report_incorrect_result() - return + for i in range(self.datasize): + if i + self.counter - 1 != int(results[i][0]): + self.report_incorrect_result() + return class TestDelete(TestDML): datasize = 100000 @@ -161,13 +171,15 @@ def loop_step(self): def verify(self, conn): sql = ''' select c1 from {tablename} order by c1; - '''.format(tablename=self.tablename, counter=self.counter) - results = dbconn.query(conn, sql).fetchall() - - for i in range(self.counter, self.datasize): - if i != int(results[i - self.counter][0]): - self.report_incorrect_result() - return + '''.format(tablename=self.tablename) + with conn.cursor() as cur: + cur.execute(sql) + results = cur.fetchall() + + for i in range(self.counter, self.datasize): + if i != int(results[i - self.counter][0]): + self.report_incorrect_result() + return # for test only if __name__ == '__main__': diff --git a/gpMgmt/test/behave_utils/utils.py b/gpMgmt/test/behave_utils/utils.py index d0f367e97361..55beb06db65f 100644 --- a/gpMgmt/test/behave_utils/utils.py +++ b/gpMgmt/test/behave_utils/utils.py @@ -11,15 +11,13 @@ import subprocess import difflib -import pg - from contextlib import closing from datetime import datetime from gppylib.commands.base import Command, ExecutionError, REMOTE from gppylib.commands.gp import chk_local_db_running, get_coordinatordatadir from gppylib.db import dbconn from gppylib.gparray import GpArray, MODE_SYNCHRONIZED - +from gppylib.utils import escape_string PARTITION_START_DATE = '2010-01-01' PARTITION_END_DATE = '2013-01-01' @@ -317,14 +315,14 @@ def check_table_exists(context, dbname, table_name, table_type=None, host=None, FROM pg_class c, pg_namespace n WHERE c.relname = '%s' AND n.nspname = '%s' AND c.relnamespace = n.oid; """ - SQL = SQL_format % (escape_string(tablename, conn=conn), escape_string(schemaname, conn=conn)) + SQL = SQL_format % (escape_string(tablename), escape_string(schemaname)) else: SQL_format = """ SELECT oid, relkind, relam, reloptions \ FROM pg_class \ WHERE relname = E'%s';\ """ - SQL = SQL_format % (escape_string(table_name, conn=conn)) + SQL = SQL_format % (escape_string(table_name)) table_row = None try: @@ -772,11 +770,6 @@ def replace_special_char_env(str): str = str.replace("$%s" % var, os.environ[var]) return str - -def escape_string(string, conn): - return pg.DB(db=conn).escape_string(string) - - def wait_for_unblocked_transactions(context, num_retries=150): """ Tries once a second to successfully commit a transaction to the database diff --git a/gpcontrib/gp_replica_check/gp_replica_check.py b/gpcontrib/gp_replica_check/gp_replica_check.py index 230f7a27849f..b0588966017f 100755 --- a/gpcontrib/gp_replica_check/gp_replica_check.py +++ b/gpcontrib/gp_replica_check/gp_replica_check.py @@ -39,8 +39,7 @@ import pipes # for shell-quoting, pipes.quote() import os from collections import defaultdict -from pg import DB - +import psycopg2 def run_sql(sql, host=None, port=None, dbname="postgres", is_query=True, @@ -50,10 +49,15 @@ def run_sql(sql, host=None, port=None, if port is None: port = int(os.getenv("PGPORT")) opt = "-c gp_role=utility" if is_utility else None - with DB(dbname=dbname, host=host, port=port, opt=opt) as db: - r = db.query(sql) - if is_query: - return r.getresult() + try: + with psycopg2.connect(dbname=dbname, host=host, port=port, options=opt) as conn: + with conn.cursor() as cur: + cur.execute(sql) + if is_query: + resultList = cur.fetchall() + return resultList + except Exception as e: + print('Exception: %s while running query %s dbname = %s' % (e, sql, dbname)) class ReplicaCheck(threading.Thread): diff --git a/python-dependencies.txt b/python-dependencies.txt index 9e63afef5289..941857c4a28e 100644 --- a/python-dependencies.txt +++ b/python-dependencies.txt @@ -1,3 +1,4 @@ psutil==5.7.0 pygresql==5.2 pyyaml==5.3.1 +psycopg2==2.9.6 From fad26c5866e96ef7bd7464bd5d7b2e6b5ecac5ed Mon Sep 17 00:00:00 2001 From: Wu Ning Date: Tue, 12 Sep 2023 10:44:13 +0800 Subject: [PATCH 02/32] Support Rocky9/OEL9/RHEL9 platform for GP7 (#16371) * Fully support Rocky9 platform for GP7 * Align the format of exttable outfile to Rocky9 * Fix compile_gpdb_clients_windows failed * Fix gpMgmt tests failure to support rocky9/oel9/rhel9 * Fix gpMgmt/bin/gpload_test/gpload2 tests on Rocky9 Authored-by: Ning Wu --- concourse/pipelines/gen_pipeline.py | 47 +- concourse/pipelines/gpdb_main-generated.yml | 171 ++++- concourse/pipelines/templates/gpdb-tpl.yml | 676 ++++++++++++++---- .../scripts/compile_gpdb_remote_windows.bash | 3 + contrib/extprotocol/expected/exttableext.out | 10 +- contrib/extprotocol/sql/exttableext.sql | 8 + gpAux/Makefile | 15 +- gpMgmt/bin/gpload_test/Makefile | 2 +- gpMgmt/bin/gppylib/commands/gp.py | 4 +- .../test/unit/test_unit_recovery_base.py | 13 +- 10 files changed, 794 insertions(+), 155 deletions(-) diff --git a/concourse/pipelines/gen_pipeline.py b/concourse/pipelines/gen_pipeline.py index 83632b9ea9c2..ec3d528fc5b2 100755 --- a/concourse/pipelines/gen_pipeline.py +++ b/concourse/pipelines/gen_pipeline.py @@ -153,12 +153,51 @@ def create_pipeline(args, git_remote, git_branch): test_trigger = "false" variables_type = args.pipeline_target + os_username = { + "rhel8" : "rhel", + "rocky8" : "rocky", + "oel8" : "oel", + "rhel9" : "rhel", + "rocky9" : "rocky", + "oel9" : "oel" + } + test_os = { + "rhel8" : "centos", + "rocky8": "centos", + "oel8" : "centos", + "rhel9" : "centos", + "rocky9": "centos", + "oel9" : "centos" + } + compile_platform = { + "rhel8" : "rocky8", + "rocky8": "rocky8", + "oel8" : "rocky8", + "rhel9" : "rocky9", + "rocky9": "rocky9", + "oel9" : "rocky9" + } + dist = { + "rhel8" : "el8", + "rocky8" : "el8", + "oel8" : "el8", + "rhel9" : "el9", + "rocky9" : "el9", + "oel9": "el9" + } + context = { 'template_filename': args.template_filename, 'generator_filename': os.path.basename(__file__), 'timestamp': datetime.datetime.now(), - 'os_types': args.os_types, + 'os_type': args.os_type, + 'default_os_type': default_os_type, + 'os_username': os_username[args.os_type], + 'test_os': test_os[args.os_type], + 'compile_platform': compile_platform[args.os_type], + 'dist': dist[args.os_type], + 'pipeline_target': args.pipeline_target, 'test_sections': args.test_sections, 'pipeline_configuration': args.pipeline_configuration, 'test_trigger': test_trigger, @@ -397,7 +436,11 @@ def main(): exit(1) if args.pipeline_target == 'prod' and not args.directed_release: - args.pipeline_configuration = 'prod' + args.test_sections = [ + 'icw', + 'cli', + 'release' + ] # use_ICW_workers adds tags to the specified concourse definitions which # correspond to dedicated concourse workers to increase performance. diff --git a/concourse/pipelines/gpdb_main-generated.yml b/concourse/pipelines/gpdb_main-generated.yml index b82930522fa9..89c7abb774ea 100644 --- a/concourse/pipelines/gpdb_main-generated.yml +++ b/concourse/pipelines/gpdb_main-generated.yml @@ -12,10 +12,10 @@ ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -t prod). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2022-11-14 22:19:02.574098 +## Generated by gen_pipeline.py at: 2023-09-12 10:40:32.386447 ## Template file: gpdb-tpl.yml -## OS Types: ['rhel8', 'win'] -## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions'] +## OS Types: +## Test Sections: ['icw', 'cli', 'release'] ## ====================================================================== ## ====================================================================== @@ -931,6 +931,45 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: test_upgrade + file: gpdb_src/concourse/tasks/test_upgrade.yml + input_mapping: + sqldump: icw_planner_rocky8_dump + params: + NUMBER_OF_NODES: 2 + +- name: icw_planner_resgroup_rocky8 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_icw_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-32 + number_of_nodes: 1 + segments_per_host: 3 + disk_size: 800 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + DATA_MOUNT_DIR: "/home/gpadmin" + CREATE_DATA_DIR: false - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: ccp-image @@ -959,9 +998,11 @@ jobs: passed: - compile_gpdb_rhel8 trigger: true - - get: bin_gpdb_rhel8 + - get: bin_gpdb + params: + skip_download: 'true' passed: - - compile_gpdb_rhel8 + - gate_compile_end - name: check_centos plan: @@ -1031,6 +1072,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpmovemirrors_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1229,6 +1272,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpstate_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1315,6 +1360,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpactivatestandby_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1375,6 +1422,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpinitstandby_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1521,6 +1570,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpcheckperf_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1581,6 +1632,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gprecoverseg_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1658,6 +1711,8 @@ jobs: head -n -2 cluster_env_files/hostfile_init_orig > cluster_env_files/hostfile_init - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gprecoverseg_newhost_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1720,6 +1775,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpaddmirrors_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1780,6 +1837,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpconfig_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1840,6 +1899,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpssh-exkeys_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1900,6 +1961,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpstart_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1960,6 +2023,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpstop_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1984,6 +2049,85 @@ jobs: BEHAVE_FLAGS: --tags=gpstop --tags=~concourse_cluster,demo_cluster +- name: resource_group_rocky8 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml + image: ccp-image + params: + TEST_OS: rocky8 + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +- name: resource_group_v2_rocky8 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + PLATFORM: rocky8-cgroup2-gpdb7 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: rocky8-cgroup2-gpdb7 + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup_v2.yml + image: ccp-image + params: + TEST_OS: rocky8 + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed - name: cli_cross_subnet plan: @@ -2113,7 +2257,8 @@ jobs: - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml params: - GPINIT_STANDBY_MASTER: '-s mdw-2' + GPINIT_STANDBY_COORDINATOR: '-s cdw-2' + <<: *default_platform - task: cross_subnet_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: ccp-image @@ -2267,7 +2412,7 @@ jobs: <<: *set_failed - # end CLI test_sections + # end cli test_sections ## ====================================================================== ## ____ _ @@ -2452,15 +2597,15 @@ jobs: RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) - in_parallel: steps: - - put: bin_gpdb_rhel8_rc - params: - file: "release_candidates/server-rc-*rhel8*.tar.gz" - - put: bin_gpdb_clients_rhel8_rc - params: - file: "release_candidates/clients-rc-*rhel8*.tar.gz" - put: server_src_rc params: file: "server_tar/*server-src-rc-*.tar.gz" + - put: bin_gpdb_rc + params: + file: "release_candidates/server-rc-*el8*.tar.gz" + - put: bin_gpdb_clients_rc + params: + file: "release_candidates/clients-rc-*el8*.tar.gz" - name: Build_Release_Candidate_RPMs diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index a87e5aa1df10..c8ca7f2c58a8 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -203,6 +203,8 @@ anchors: groups: - name: all jobs: + - gate_compile_start + {% if os_type == compile_platform %} - check_format {% if "rhel8" in os_types %} - compile_gpdb_rhel8 @@ -264,17 +266,21 @@ groups: {%endif %} -{% if pipeline_configuration == "prod" or build_test_rc_rpm or directed_release %} +{% if ( ("release" in test_sections and os_type == compile_platform) or build_test_rc_rpm or directed_release) %} ## ====================================================================== - name: Release jobs: -{% if pipeline_configuration == "prod" or directed_release %} +{% if "release" in test_sections or directed_release %} + - gate_icw_end + - gate_cli_end - gate_release_candidate_start - Release_Candidate {% if not directed_release %} - Publish Server Builds {% endif %} +{% if not directed_release %} + - build_release_candidate_rpms {% endif %} {% if not directed_release %} - Build_Release_Candidate_RPMs @@ -285,11 +291,13 @@ groups: - name: Compile jobs: -{% if "rhel8" in os_types %} - - compile_gpdb_rhel8 - - test_gpdb_clients_rhel8 -{% endif %} -{% if "win" in os_types %} + - gate_compile_start + - compile_gpdb_[[ compile_platform ]] + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - prepare_binary_swap_gpdb_[[ os_type ]] + {% endif %} + - test_gpdb_clients_[[ os_type ]] + {% if os_type == compile_platform %} - compile_gpdb_clients_windows {% endif %} @@ -470,7 +478,17 @@ resources: username: _json_key password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) -- name: gpdb7-rhel8-test +{% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} +- name: previous_gpdb + type: tanzunet + source: + api_token: ((pivotal-gpdb-pivnet-api-token)) + product_slug: pivotal-gpdb + product_version: ((pivotal-gpdb-pivnet-product-version)) +{% endif %} + +{% if os_type != compile_platform %} +- name: gpdb7-[[ compile_platform ]]-build type: registry-image source: repository: gcr.io/data-gpdb-private-images/gpdb7-rhel8-test @@ -481,7 +499,7 @@ resources: {% if pipeline_configuration == "prod" or directed_release %} {% if not directed_release %} -- name: bin_gpdb_rhel8_icw_green +- name: bin_gpdb_icw_green type: s3 source: access_key_id: ((aws-bucket-access-key-id)) @@ -490,20 +508,20 @@ resources: secret_access_key: ((aws-bucket-secret-access-key)) versioned_file: bin_gpdb_rhel8/gpdb_branch_master/icw_green/bin_gpdb.tar.gz -- name: bin_gpdb_rhel8_rc +{% if os_type == compile_platform %} +- name: bin_gpdb_rc type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/server-rc-(.*)-rhel8_x86_64((rc-build-type-gcs)).tar.gz + regexp: server/published/main/server-rc-(.*)-[[ dist ]]_x86_64((rc-build-type-gcs)).tar.gz -- name: bin_gpdb_clients_rhel8_rc +- name: bin_gpdb_clients_rc type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: clients/published/main/clients-rc-(.*)-rhel8_x86_64((rc-build-type-gcs)).tar.gz - + regexp: clients/published/main/clients-rc-(.*)-[[ dist ]]_x86_64((rc-build-type-gcs)).tar.gz {% endif %} - name: server_src_rc type: gcs @@ -517,13 +535,13 @@ resources: {% endif %} {% endif %} -{% if pipeline_configuration == "prod" or build_test_rc_rpm %} -- name: gpdb_rpm_installer_rhel8 +{% if ( ("release" in test_sections and os_type == compile_platform) or build_test_rc_rpm) %} +- name: gpdb_rpm_installer_[[ os_type ]] type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/greenplum-db-(.*)-rhel8-x86_64((rc-build-type-gcs)).rpm + regexp: server/published/main/greenplum-db-(.*)-[[ dist ]]-x86_64((rc-build-type-gcs)).rpm - name: greenplum-database-release type: git @@ -549,25 +567,33 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_rhel8/bin_gpdb.tar.gz -- name: bin_gpdb_clients_rhel8 +{% if compile_platform == default_os_type %} +- name: llvm-with-asserts-packages + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/[[ default_os_type ]]/llvm-with-asserts-(.*)-[[ dist ]]_x86_64.tar.gz +{% endif %} + +{% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} +- name: binary_swap_gpdb_[[ os_type ]] type: gcs source: bucket: ((gcs-bucket-intermediates)) json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_clients_rhel8/bin_gpdb_clients.tar.gz -{% if pipeline_configuration == "prod" %} -- name: server-build-rhel8 +{% if os_type == compile_platform %} +- name: server-build-[[ os_type ]] type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/server-build-(.*)-rhel8_x86_64((rc-build-type-gcs)).tar.gz -{% endif %} - + regexp: server/published/main/server-build-(.*)-[[ dist ]]_x86_64((rc-build-type-gcs)).tar.gz {% endif %} -{% if "win" in os_types %} +{% if os_type == compile_platform %} - name: bin_gpdb_clients_windows type: gcs source: @@ -606,6 +632,31 @@ jobs: ## |_| ## ====================================================================== +## ====================================================================== +## ____ _ _ +## / ___|___ _ __ ___ _ __ (_) | ___ +## | | / _ \| '_ ` _ \| '_ \| | |/ _ \ +## | |__| (_) | | | | | | |_) | | | __/ +## \____\___/|_| |_| |_| .__/|_|_|\___| +## |_| +## ====================================================================== + +- name: gate_compile_start + plan: + - get: gpdb_src + {% if os_type != compile_platform and pipeline_target == "prod" %} + trigger: false + {% else %} + trigger: true + {% endif %} + params: + skip_download: 'true' + {% if os_type != compile_platform and pipeline_target == "prod" %} + - get: reduced-frequency-trigger + trigger: true + {% endif %} + +{% if os_type == compile_platform %} - name: check_format plan: - get: gpdb_src @@ -637,9 +688,16 @@ jobs: trigger: ((reduced-frequency-trigger-flag)) - get: gpdb_src passed: + {% if os_type == compile_platform %} - check_format - trigger: ((gpdb_src-trigger-flag)) - - get: gpdb7-rhel8-build + {% else %} + - gate_compile_start + {% endif %} + trigger: true + - get: gpdb7-[[ compile_platform ]]-build + {% if compile_platform == default_os_type %} + - get: llvm-with-asserts-packages + {% endif %} - task: compile_gpdb image: gpdb7-rhel8-build file: gpdb_src/concourse/tasks/compile_gpdb.yml @@ -656,7 +714,28 @@ jobs: params: file: gpdb_artifacts/bin_gpdb_clients.tar.gz -- name: test_gpdb_clients_rhel8 +{% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} +- name: prepare_binary_swap_gpdb_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + trigger: true + passed: [compile_gpdb_[[ compile_platform ]]] + - get: gpdb_package + resource: previous_gpdb + params: + globs: [greenplum-db-7*-el8-x86_64.rpm] + - get: gpdb7-[[ compile_platform ]]-build + - task: generate_previous_bin_gpdb + file: gpdb_src/concourse/tasks/extract_package.yml + image: gpdb7-[[ compile_platform ]]-build + - put: binary_swap_gpdb_[[ os_type ]] + params: + file: gpdb_artifacts/bin_gpdb.tar.gz +{% endif %} + +- name: test_gpdb_clients_[[ os_type ]] plan: - in_parallel: steps: @@ -674,8 +753,7 @@ jobs: image: gpdb7-rhel8-test file: gpdb_src/concourse/tasks/test_gpdb_clients.yml -{% endif %} -{% if "win" in os_types %} +{% if os_type == compile_platform %} - name: compile_gpdb_clients_windows serial: true plan: @@ -683,10 +761,12 @@ jobs: steps: - get: gpdb_src trigger: true - - get: gpdb7-rhel8-build + passed: + - check_format + - get: gpdb7-[[ compile_platform ]]-build - task: compile_gpdb_windows_remote file: gpdb_src/concourse/tasks/compile_gpdb_remote_windows.yml - image: gpdb7-rhel8-build + image: gpdb7-[[ compile_platform ]]-build params: REMOTE_HOST: ((remote_win_host_build)) REMOTE_PORT: ((remote_win_port_build)) @@ -695,8 +775,92 @@ jobs: - put: bin_gpdb_clients_windows params: file: "gpdb_artifacts/greenplum-clients-x86_64.tar.gz" + +- name: publish_server_builds + old_name: Publish Server Builds + plan: + - in_parallel: + steps: + - get: gpdb_src + trigger: true + passed: + - compile_gpdb_[[ compile_platform ]] + - get: bin_gpdb + passed: [compile_gpdb_[[ compile_platform ]]] + - get: gpdb7-[[ compile_platform ]]-build + - task: rename_server_build_artifacts + image: gpdb7-[[ compile_platform ]]-build + config: + platform: linux + inputs: + - name: gpdb_src + - name: bin_gpdb + outputs: + - name: output + params: + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) + run: + path: bash + args: + - -ec + - | + server_version="$(./gpdb_src/getversion --short)" + cp bin_gpdb/bin_gpdb.tar.gz "output/server-build-${server_version}-[[ dist ]]_x86_64${RC_BUILD_TYPE_GCS}.tar.gz" + - in_parallel: + steps: + - put: server-build-[[ os_type ]] + params: + file: output/server-build-*-[[ dist ]]*.tar.gz + {% endif %} -{% if "ICW" in test_sections %} + +- name: gate_compile_end + plan: + - get: gpdb_src + trigger: true + params: + skip_download: 'true' + passed: + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - prepare_binary_swap_gpdb_[[ os_type ]] + {% endif %} + {% if os_type == compile_platform %} + - compile_gpdb_clients_windows + - publish_server_builds + {% endif %} + - test_gpdb_clients_[[ os_type ]] + - get: bin_gpdb + params: + skip_download: 'true' + passed: + - test_gpdb_clients_[[ os_type ]] + {% if os_type == compile_platform %} + - publish_server_builds + {% endif %} + - get: bin_gpdb_clients + params: + skip_download: 'true' + passed: + - test_gpdb_clients_[[ os_type ]] + {% if os_type == compile_platform %} + - get: bin_gpdb_clients_windows + params: + skip_download: 'true' + passed: + - compile_gpdb_clients_windows + - get: server-build-[[ os_type ]] + params: + skip_download: 'true' + passed: [publish_server_builds] + {% endif %} + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb_[[ os_type ]] + params: + skip_download: 'true' + passed: [ prepare_binary_swap_gpdb_[[ os_type ]] ] + {% endif %} + +{% if "icw" in test_sections %} ## ====================================================================== ## ___ ______ __ ## |_ _/ ___\ \ / / @@ -704,8 +868,31 @@ jobs: ## | | |___ \ V V / ## |___\____| \_/\_/ ## ====================================================================== -{% if "rhel8" in os_types %} -- name: gpdb_pitr_rhel8 +- name: gate_icw_start + plan: + - in_parallel: + steps: + - get: bin_gpdb + params: + skip_download: 'true' + passed: [gate_compile_end] + trigger: true + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb_[[ os_type ]] + params: + skip_download: 'true' + passed: [gate_compile_end] + {% endif %} + - get: gpdb_src + params: + skip_download: 'true' + passed: [gate_compile_end] + - get: bin_gpdb_clients + params: + skip_download: 'true' + passed: [ gate_compile_end ] + +- name: gpdb_pitr_[[ os_type ]] plan: - in_parallel: steps: @@ -785,12 +972,50 @@ jobs: tags: [icw-rhel8] {% endif %} file: gpdb_src/concourse/tasks/ic_gpdb.yml - image: gpdb7-rhel8-test + image: gpdb7-[[ os_type ]]-test + timeout: 24h + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on -c jit=off' installcheck-world + TEST_OS: [[ test_os ]] + CONFIGURE_FLAGS: ((configure_flags)) + BLDWRAP_POSTGRES_CONF_ADDONS: + - optimizer_use_gpdb_allocators=off + +- name: icw_planner_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: bin_gpdb + passed: [gate_icw_start] + trigger: true + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb + passed: [gate_icw_start] + resource: binary_swap_gpdb_[[ os_type ]] + {% endif %} + - get: gpdb7-[[ os_type ]]-test + {% if compile_platform == default_os_type %} + - get: llvm-with-asserts-packages + {% endif %} + - task: ic_gpdb + {% if use_ICW_workers %} + tags: [icw-worker] + {% endif %} + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + file: gpdb_src/concourse/tasks/ic_gpdb_binary_swap.yml + {% else %} + file: gpdb_src/concourse/tasks/ic_gpdb.yml + {% endif %} + image: gpdb7-[[ os_type ]]-test timeout: 24h params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off -c jit=off' installcheck-world - TEST_OS: centos + TEST_OS: [[ test_os ]] + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} TEST_BINARY_SWAP: ((test-binary-swap)) + {% endif %} CONFIGURE_FLAGS: ((configure_flags)) DUMP_DB: "true" - put: icw_planner_rhel8_dump @@ -804,20 +1029,34 @@ jobs: - get: gpdb_src passed: [compile_gpdb_rhel8] - get: bin_gpdb - passed: [compile_gpdb_rhel8] - resource: bin_gpdb_rhel8 - trigger: [[ test_trigger ]] - - get: gpdb7-rhel8-test + passed: [gate_icw_start] + trigger: true + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb + passed: [gate_icw_start] + resource: binary_swap_gpdb_[[ os_type ]] + {% endif %} + - get: gpdb7-[[ os_type ]]-test + {% if compile_platform == default_os_type %} + - get: llvm-with-asserts-packages + {% endif %} - task: ic_gpdb {% if use_ICW_workers %} tags: [icw-rhel8] {% endif %} + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + file: gpdb_src/concourse/tasks/ic_gpdb_binary_swap.yml + {% else %} file: gpdb_src/concourse/tasks/ic_gpdb.yml - image: gpdb7-rhel8-test + {% endif %} + image: gpdb7-[[ os_type ]]-test timeout: 24h params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off -c jit=on -c jit_above_cost=0 -c gp_explain_jit=off' installcheck - TEST_OS: centos + TEST_OS: [[ test_os ]] + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + TEST_BINARY_SWAP: ((test-binary-swap)) + {% endif %} CONFIGURE_FLAGS: ((configure_flags)) - name: icw_gporca_icproxy_rhel8 @@ -1063,6 +1302,88 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: test_upgrade + file: gpdb_src/concourse/tasks/test_upgrade.yml + input_mapping: + sqldump: icw_planner_[[ os_type ]]_dump + params: + NUMBER_OF_NODES: 2 + +- name: icw_planner_resgroup_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_icw_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-32 + number_of_nodes: 1 + segments_per_host: 3 + disk_size: 800 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + DATA_MOUNT_DIR: "/home/gpadmin" + CREATE_DATA_DIR: false + - task: run_tests + file: gpdb_src/concourse/tasks/ic_resgroup.yml + image: ccp-image + params: + TEST_OS: [[ os_type ]] + MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off -c jit=off' installcheck-world + BLDWRAP_POSTGRES_CONF_ADDONS: + - gp_resource_manager=group + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +- name: icw_gporca_resgroup_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_icw_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-32 + number_of_nodes: 1 + segments_per_host: 3 + disk_size: 800 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + DATA_MOUNT_DIR: "/home/gpadmin" + CREATE_DATA_DIR: false - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: ccp-image @@ -1073,6 +1394,61 @@ jobs: ensure: <<: *set_failed +- name: gate_icw_end + plan: + - in_parallel: + steps: + - get: bin_gpdb + passed: + - icw_gporca_[[ os_type ]] + - icw_gporca_jit_[[ os_type ]] + - icw_gporca_memory_[[ os_type ]] + - icw_planner_jit_[[ os_type ]] + - icw_gporca_icproxy_[[ os_type ]] + - icw_planner_icproxy_[[ os_type ]] + - icw_gporca_ictcp_[[ os_type ]] + - icw_mirrorless_[[ os_type ]] + - icw_extensions_gpcloud_[[ os_type ]] + - unit_tests_gporca_[[ os_type ]] + - gpdb_pitr_[[ os_type ]] + - gpexpand_[[ os_type ]] + - pg_upgrade_[[ os_type ]] + - interconnect_[[ os_type ]] + - get: icw_planner_[[ os_type ]]_dump + params: + skip_download: 'true' + passed: + - pg_upgrade_[[ os_type ]] + - gpexpand_[[ os_type ]] + - get: gpdb_src + params: + skip_download: 'true' + passed: + - icw_gporca_[[ os_type ]] + - icw_gporca_jit_[[ os_type ]] + - icw_gporca_memory_[[ os_type ]] + - icw_planner_jit_[[ os_type ]] + - icw_gporca_icproxy_[[ os_type ]] + - icw_planner_icproxy_[[ os_type ]] + - icw_gporca_ictcp_[[ os_type ]] + - icw_mirrorless_[[ os_type ]] + - icw_extensions_gpcloud_[[ os_type ]] + - unit_tests_gporca_[[ os_type ]] + - gpdb_pitr_[[ os_type ]] + - gpexpand_[[ os_type ]] + - pg_upgrade_[[ os_type ]] + - icw_planner_resgroup_[[ os_type ]] + - icw_gporca_resgroup_[[ os_type ]] + - interconnect_[[ os_type ]] + trigger: true + - get: bin_gpdb_clients + params: + skip_download: 'true' + passed: [unit_tests_gporca_[[ os_type ]]] +{% if ("release" in test_sections and os_type == compile_platform) %} + - put: bin_gpdb_icw_green + params: + file: bin_gpdb/bin_gpdb.tar.gz {% endif %} {% endif %} @@ -1096,7 +1472,7 @@ jobs: trigger: true - get: bin_gpdb_rhel8 passed: - - compile_gpdb_rhel8 + - gate_compile_end - name: check_centos plan: @@ -1188,6 +1564,8 @@ jobs: {% endif %} - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: [[ test.name ]]_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1220,6 +1598,86 @@ jobs: {% endif %} {% endfor %} +- name: resource_group_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml + image: ccp-image + params: + TEST_OS: [[ os_type ]] + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +- name: resource_group_v2_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + PLATFORM: [[ os_type ]]-cgroup2-gpdb7 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: [[ os_type ]]-cgroup2-gpdb7 + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup_v2.yml + image: ccp-image + params: + TEST_OS: [[ os_type ]] + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + - name: cli_cross_subnet plan: - in_parallel: @@ -1348,7 +1806,8 @@ jobs: - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml params: - GPINIT_STANDBY_MASTER: '-s mdw-2' + GPINIT_STANDBY_COORDINATOR: '-s cdw-2' + <<: *default_platform - task: cross_subnet_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: ccp-image @@ -1488,9 +1947,7 @@ jobs: {% endif %} -{% endif %} # end CLI test_sections - -{% if pipeline_configuration == "prod" or build_test_rc_rpm or directed_release %} +{% if ( ("release" in test_sections and os_type == compile_platform ) or build_test_rc_rpm or directed_release) %} ## ====================================================================== ## ____ _ ## | _ \ ___| | ___ __ _ ___ ___ @@ -1507,60 +1964,16 @@ jobs: - get: gpdb_src trigger: true passed: - - check_format - - compile_gpdb_rhel8 - - icw_gporca_rhel8 - - icw_gporca_jit_rhel8 - - icw_planner_rhel8 - - icw_planner_jit_rhel8 - - icw_gporca_icproxy_rhel8 - - icw_planner_icproxy_rhel8 - - icw_gporca_ictcp_rhel8 - - icw_mirrorless_rhel8 - - icw_extensions_gpcloud_rhel8 - - cli_cross_subnet - - unit_tests_gporca_rhel8 - - test_gpdb_clients_rhel8 - - gpdb_pitr_rhel8 -{% for test in CLI_BEHAVE_TESTS %} -{% if "rhel8" in os_types %} - - [[ test.name ]]_rhel8 - - resource_group_rhel8 - - interconnect_rhel8 -{% endif %} -{% endfor %} - - pg_upgrade_rhel8 - - gpexpand_rhel8 - - check_centos - - get: bin_gpdb_rhel8 + - gate_icw_end + - gate_cli_end + - get: bin_gpdb + params: + skip_download: 'true' trigger: true passed: - - compile_gpdb_rhel8 - - icw_planner_rhel8 - - icw_planner_jit_rhel8 - - icw_gporca_rhel8 - - icw_gporca_jit_rhel8 - - icw_gporca_icproxy_rhel8 - - icw_planner_icproxy_rhel8 - - icw_gporca_ictcp_rhel8 - - icw_mirrorless_rhel8 - - icw_extensions_gpcloud_rhel8 - - resource_group_rhel8 - - interconnect_rhel8 - - cli_cross_subnet - {% for test in CLI_BEHAVE_TESTS %} - - [[ test.name ]]_rhel8 -{% endfor %} - - get: bin_gpdb_clients_rhel8 - passed: - - compile_gpdb_rhel8 - -{% if not directed_release %} -- name: Publish Server Builds - plan: - - in_parallel: - steps: - - get: gpdb_src + - gate_icw_end + - gate_cli_end + - get: bin_gpdb_clients trigger: true passed: - compile_gpdb_rhel8 @@ -1576,10 +1989,12 @@ jobs: steps: - put: server-build-rhel8 params: - file: output/server-build-*-rhel8*.tar.gz + skip_download: 'true' + passed: + - gate_icw_end -{% endif %} -- name: Release_Candidate +- name: release_candidate + old_name: Release_Candidate plan: - in_parallel: steps: @@ -1599,10 +2014,25 @@ jobs: steps: {% if not directed_release %} - task: rename_rc_artifacts - image: gpdb7-rhel8-build - file: gpdb_src/concourse/tasks/rename_rc_artifacts.yml - params: - RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) + image: gpdb7-[[ compile_platform ]]-build + config: + platform: linux + inputs: + - name: gpdb_src + - name: bin_gpdb + - name: bin_gpdb_clients + outputs: + - name: release_candidates + params: + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) + run: + path: bash + args: + - -ec + - | + gpdb_semver=$(gpdb_src/getversion | cut -d' ' -f1) + cp -v bin_gpdb/bin_gpdb.tar.gz release_candidates/server-rc-${gpdb_semver}-[[ dist ]]_x86_64${RC_BUILD_TYPE_GCS}.tar.gz + cp -v bin_gpdb_clients/bin_gpdb_clients.tar.gz release_candidates/clients-rc-${gpdb_semver}-[[ dist ]]_x86_64${RC_BUILD_TYPE_GCS}.tar.gz - task: verify_gpdb_versions image: gpdb7-rhel8-build file: gpdb_src/concourse/tasks/verify_gpdb_versions.yml @@ -1620,21 +2050,23 @@ jobs: - in_parallel: steps: {% if not directed_release %} - - put: bin_gpdb_rhel8_rc - params: - file: "release_candidates/server-rc-*rhel8*.tar.gz" - - put: bin_gpdb_clients_rhel8_rc - params: - file: "release_candidates/clients-rc-*rhel8*.tar.gz" -{% endif %} + {% if os_type == default_os_type %} - put: server_src_rc params: file: "server_tar/*server-src-rc-*.tar.gz" - + {% endif %} + - put: bin_gpdb_rc + params: + file: "release_candidates/server-rc-*[[ dist ]]*.tar.gz" + - put: bin_gpdb_clients_rc + params: + file: "release_candidates/clients-rc-*[[ dist ]]*.tar.gz" +{% endif %} {% endif %} -{% if not directed_release %} -- name: Build_Release_Candidate_RPMs +{% if not directed_release and os_type == compile_platform %} +- name: build_release_candidate_rpms + old_name: Build_Release_Candidate_RPMs plan: - in_parallel: steps: @@ -1650,14 +2082,12 @@ jobs: {% if not build_test_rc_rpm %} passed: [gate_release_candidate_start] {% endif %} - - get: gpdb7-rhel8-build + - get: gpdb7-[[ compile_platform ]]-build - get: greenplum-database-release - get: license_file - task: create_gpdb_rpm_package_rhel8 file: greenplum-database-release/ci/concourse/tasks/build-gpdb-rpm.yml - image: gpdb7-rhel8-build - input_mapping: - bin_gpdb: bin_gpdb_rhel8 + image: gpdb7-[[ compile_platform ]]-build output_mapping: gpdb_rpm_installer: gpdb_rpm_rhel8 params: @@ -1668,7 +2098,7 @@ jobs: GPDB_URL: https://github.com/greenplum-db/gpdb GPDB_OSS: false - task: rename_rc_artifacts - image: gpdb7-rhel8-build + image: gpdb7-[[ compile_platform ]]-build config: platform: linux inputs: @@ -1687,9 +2117,9 @@ jobs: {% if build_test_rc_rpm %} # Uniquely name the rpm using the github username and test branch # to prevent filename collisions on GCS. - cp -v gpdb_rpm_rhel8/*.rpm renamed_gpdb_rpm_rhel8/greenplum-db-[[ git_username ]]-[[ git_branch ]]-${gpdb_semver}-rhel8-x86_64${RC_BUILD_TYPE_GCS}.rpm + cp -v gpdb_rpm_[[ os_type ]]/*.rpm renamed_gpdb_rpm_[[ os_type ]]/greenplum-db-[[ git_username ]]-[[ git_branch ]]-${gpdb_semver}-[[ dist ]]-x86_64${RC_BUILD_TYPE_GCS}.rpm {% else %} - cp -v gpdb_rpm_rhel8/*.rpm renamed_gpdb_rpm_rhel8/greenplum-db-${gpdb_semver}-rhel8-x86_64${RC_BUILD_TYPE_GCS}.rpm + cp -v gpdb_rpm_[[ os_type ]]/*.rpm renamed_gpdb_rpm_[[ os_type ]]/greenplum-db-${gpdb_semver}-[[ dist ]]-x86_64${RC_BUILD_TYPE_GCS}.rpm {% endif %} - in_parallel: steps: diff --git a/concourse/scripts/compile_gpdb_remote_windows.bash b/concourse/scripts/compile_gpdb_remote_windows.bash index 3cb7807f0502..ba76af9c4e99 100755 --- a/concourse/scripts/compile_gpdb_remote_windows.bash +++ b/concourse/scripts/compile_gpdb_remote_windows.bash @@ -66,6 +66,9 @@ EOF } function download() { + # On RHEL9 platform, the path C:\Users\buildbot cannot be escaped for scp + # So replace all backslash with double backslashes + WORK_DIR=${WORK_DIR//\\/\\\\} pushd "$ROOT_DIR/gpdb_artifacts/" scp -P "${REMOTE_PORT}" -q "${REMOTE_USER}"@"${REMOTE_HOST}":"${WORK_DIR}/*.msi" ./ scp -P "${REMOTE_PORT}" -q "${REMOTE_USER}"@"${REMOTE_HOST}":"${WORK_DIR}/*.exe" ./ diff --git a/contrib/extprotocol/expected/exttableext.out b/contrib/extprotocol/expected/exttableext.out index 7c68cb2c80ec..04faefe2c11d 100644 --- a/contrib/extprotocol/expected/exttableext.out +++ b/contrib/extprotocol/expected/exttableext.out @@ -1559,6 +1559,14 @@ ERROR: protocol attribute "validatorproc" not recognized drop role if exists demoprot_nopriv; create role demoprot_nopriv with login ; NOTICE: resource queue required -- using default resource queue "pg_default" +-- start_matchsubs +-- m/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ +-- s/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ERROR: internal error: demoprot called with a different protocol \(demoprot_new\)/ +-- end_matchsubs +-- start_matchsubs +-- m/line \d+ of file/ +-- s/line \d+ of file/file/ +-- end_matchsubs -- Test 92: Rename existing protocol DROP FUNCTION IF EXISTS url_validator(); CREATE OR REPLACE FUNCTION url_validator() RETURNS void AS @@ -1630,7 +1638,7 @@ ERROR: protocol "demoprot" does not exist (seg0 slice1 rh55-qavm55:7532 pid=18 -- However demoprot implementation prevents using any other protocol name than "demoprot" -- therefore the error is expected. select count(*) from exttabtest_r_new; -ERROR: internal error: demoprot called with a different protocol (demoprot_new) (gpextprotocol.c:108) (seg0 slice1 rh55-qavm55:7532 pid=18394) (cdbdisp.c:1453) +ERROR: internal error: demoprot called with a different protocol (demoprot_new) CONTEXT: External table exttabtest_r_new, file demoprot_new://exttabtest.txt -- Rename protocol name back to demoprot ALTER PROTOCOL demoprot_new RENAME to demoprot; diff --git a/contrib/extprotocol/sql/exttableext.sql b/contrib/extprotocol/sql/exttableext.sql index 227475263f85..081c4665c341 100644 --- a/contrib/extprotocol/sql/exttableext.sql +++ b/contrib/extprotocol/sql/exttableext.sql @@ -1252,6 +1252,14 @@ validatorfunc = url_validator drop role if exists demoprot_nopriv; create role demoprot_nopriv with login ; +-- start_matchsubs +-- m/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ +-- s/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ERROR: internal error: demoprot called with a different protocol \(demoprot_new\)/ +-- end_matchsubs +-- start_matchsubs +-- m/line \d+ of file/ +-- s/line \d+ of file/file/ +-- end_matchsubs -- Test 92: Rename existing protocol DROP FUNCTION IF EXISTS url_validator(); diff --git a/gpAux/Makefile b/gpAux/Makefile index 410965e007c3..cb915e37f022 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -120,13 +120,14 @@ DEFPORT=5432 ORCA_CONFIG=--enable-orca -rhel6_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -rhel7_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -rhel8_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -rocky8_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -ol8_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -ubuntu18.04_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -sles12_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm +rhel6_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rhel7_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rhel8_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rocky8_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rocky9_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +ol8_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +ubuntu18.04_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +sles12_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm BLD_CONFIGFLAGS=$($(BLD_ARCH)_CONFIGFLAGS) CONFIGFLAGS=$(strip $(BLD_CONFIGFLAGS) --with-pgport=$(DEFPORT) $(BLD_DEPLOYMENT_SETTING)) diff --git a/gpMgmt/bin/gpload_test/Makefile b/gpMgmt/bin/gpload_test/Makefile index 9a6bcec1dbc9..ed8c77c49f7f 100644 --- a/gpMgmt/bin/gpload_test/Makefile +++ b/gpMgmt/bin/gpload_test/Makefile @@ -24,7 +24,7 @@ else installcheck: gpdiff.pl gpstringsubs.pl @echo "doing test in OS: "; echo $(TEST_OS) @cd gpload && ./TEST.py - @cd gpload2 && PYTHONIOENCODING=utf-8 pytest TEST_local_* + @cd gpload2 && export PGSSLMODE=disable && PYTHONIOENCODING=utf-8 pytest TEST_local_* endif clean distclean: diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index b749eb797245..c29d3cee3d67 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1009,11 +1009,11 @@ def __init__(self,name,gphome,ctxt=LOCAL,remoteHost=None): self.gphome=gphome #self.cmdStr="%s/bin/postgres --gp-version" % gphome - self.cmdStr="$GPHOME/bin/postgres --gp-version" + self.cmdStr="echo 'START_CMD_OUTPUT';$GPHOME/bin/postgres --gp-version" Command.__init__(self,name,self.cmdStr,ctxt,remoteHost) def get_version(self): - return self.results.stdout.strip() + return self.results.stdout.strip().split('START_CMD_OUTPUT\n')[1] @staticmethod def local(name,gphome): diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py b/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py index 8d7f7c7a316c..a2b0773be085 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py @@ -3,6 +3,7 @@ import io from mock import call, Mock, patch, ANY import sys +import re from .gp_unittest import GpTestCase import gppylib @@ -60,8 +61,8 @@ def _asserts_for_passing_tests(self, stderr_buf, ex, enable_verbose_count=0, war def _asserts_for_failing_tests(self, ex, stderr_buf, expected_message, info_count=1): self.assertEqual(1, ex.exception.code) - self.assertEqual(expected_message, stderr_buf.getvalue().strip()) - self.assertEqual([call(expected_message)], self.mock_logger.error.call_args_list) + self.assertRegexpMatches(expected_message, stderr_buf.getvalue().strip()) + self.assertTrue(any(re.search(expected_message, call_args[0]) for call_args, _ in self.mock_logger.error.call_args_list)) self.assertEqual(info_count, self.mock_logger.info.call_count) def _assert_workerpool_calls(self, mock_workerpool): @@ -211,9 +212,9 @@ def test_invalid_cmd_fails(self): stderr_buf, ex = self.run_recovery_base_get_stderr() self._asserts_for_failing_tests(ex, stderr_buf, - '[{"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + '[{"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null},' - ' {"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + ' {"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null}]') #TODO do we need this test where an invalid command fails but with a wrapper error? @@ -239,9 +240,9 @@ def test_invalid_cmd_verbose_fails(self): stderr_buf, ex = self.run_recovery_base_get_stderr() self._asserts_for_failing_tests(ex, stderr_buf, - '[{"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + '[{"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null},' - ' {"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + ' {"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null}]') self.assertEqual(1, self.mock_enable_verbose_logging.call_count) From f7643a2e1f31e3002a5d83d8566dc68fdd3cc750 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Sat, 25 Sep 2021 11:27:20 +0200 Subject: [PATCH 03/32] Disable OpenSSL EVP digest padding in pgcrypto The PX layer in pgcrypto is handling digest padding on its own uniformly for all backend implementations. Starting with OpenSSL 3.0.0, DecryptUpdate doesn't flush the last block in case padding is enabled so explicitly disable it as we don't use it. This will be backpatched to all supported version once there is sufficient testing in the buildfarm of OpenSSL 3. Reviewed-by: Peter Eisentraut, Michael Paquier Discussion: https://postgr.es/m/FEF81714-D479-4512-839B-C769D2605F8A@yesql.se Backpatch-through: 9.6 --- contrib/pgcrypto/openssl.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index c28cc5b4bdcc..e983a3803fe0 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -381,6 +381,8 @@ gen_ossl_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, { if (!EVP_DecryptInit_ex(od->evp_ctx, od->evp_ciph, NULL, NULL, NULL)) return PXE_CIPHER_INIT; + if (!EVP_CIPHER_CTX_set_padding(od->evp_ctx, 0)) + return PXE_CIPHER_INIT; if (!EVP_CIPHER_CTX_set_key_length(od->evp_ctx, od->klen)) return PXE_CIPHER_INIT; if (!EVP_DecryptInit_ex(od->evp_ctx, NULL, NULL, od->key, od->iv)) @@ -405,6 +407,8 @@ gen_ossl_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, { if (!EVP_EncryptInit_ex(od->evp_ctx, od->evp_ciph, NULL, NULL, NULL)) return PXE_CIPHER_INIT; + if (!EVP_CIPHER_CTX_set_padding(od->evp_ctx, 0)) + return PXE_CIPHER_INIT; if (!EVP_CIPHER_CTX_set_key_length(od->evp_ctx, od->klen)) return PXE_CIPHER_INIT; if (!EVP_EncryptInit_ex(od->evp_ctx, NULL, NULL, od->key, od->iv)) From 16857c5fc837f8ff439bee5b17125154e78f7c24 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Sat, 25 Sep 2021 11:27:28 +0200 Subject: [PATCH 04/32] Add alternative output for OpenSSL 3 without legacy loaded OpenSSL 3 introduced the concept of providers to support modularization, and moved the outdated ciphers to the new legacy provider. In case it's not loaded in the users openssl.cnf file there will be a lot of regress test failures, so add alternative outputs covering those. Also document the need to load the legacy provider in order to use older ciphers with OpenSSL-enabled pgcrypto. This will be backpatched to all supported version once there is sufficient testing in the buildfarm of OpenSSL 3. Reviewed-by: Michael Paquier Discussion: https://postgr.es/m/FEF81714-D479-4512-839B-C769D2605F8A@yesql.se Backpatch-through: 9.6 --- contrib/pgcrypto/expected/blowfish_1.out | 95 +++ contrib/pgcrypto/expected/cast5_1.out | 48 ++ contrib/pgcrypto/expected/des_1.out | 31 + contrib/pgcrypto/expected/pgp-decrypt_1.out | 421 +++++++++++ .../expected/pgp-pubkey-decrypt_1.out | 652 ++++++++++++++++++ 5 files changed, 1247 insertions(+) create mode 100644 contrib/pgcrypto/expected/blowfish_1.out create mode 100644 contrib/pgcrypto/expected/cast5_1.out create mode 100644 contrib/pgcrypto/expected/des_1.out create mode 100644 contrib/pgcrypto/expected/pgp-decrypt_1.out create mode 100644 contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out diff --git a/contrib/pgcrypto/expected/blowfish_1.out b/contrib/pgcrypto/expected/blowfish_1.out new file mode 100644 index 000000000000..565a0853cb03 --- /dev/null +++ b/contrib/pgcrypto/expected/blowfish_1.out @@ -0,0 +1,95 @@ +-- +-- Blowfish cipher +-- +-- ensure consistent test output regardless of the default bytea format +SET bytea_output TO escape; +-- some standard Blowfish testvalues +SELECT encode(encrypt( +decode('0000000000000000', 'hex'), +decode('0000000000000000', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('ffffffffffffffff', 'hex'), +decode('ffffffffffffffff', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('1000000000000001', 'hex'), +decode('3000000000000000', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('1111111111111111', 'hex'), +decode('1111111111111111', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('0123456789abcdef', 'hex'), +decode('fedcba9876543210', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('01a1d6d039776742', 'hex'), +decode('fedcba9876543210', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('ffffffffffffffff', 'hex'), +decode('0000000000000000', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- setkey +SELECT encode(encrypt( +decode('fedcba9876543210', 'hex'), +decode('f0e1d2c3b4a5968778695a4b3c2d1e0f', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- with padding +SELECT encode(encrypt( +decode('01234567890123456789', 'hex'), +decode('33443344334433443344334433443344', 'hex'), +'bf-ecb'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- cbc +-- 28 bytes key +SELECT encode(encrypt( +decode('6b77b4d63006dee605b156e27403979358deb9e7154616d959f1652bd5', 'hex'), +decode('37363534333231204e6f77206973207468652074696d6520666f7220', 'hex'), +'bf-cbc'), 'hex'); +ERROR: encrypt error: Key was too big +-- 29 bytes key +SELECT encode(encrypt( +decode('6b77b4d63006dee605b156e27403979358deb9e7154616d959f1652bd5ff92cc', 'hex'), +decode('37363534333231204e6f77206973207468652074696d6520666f722000', 'hex'), +'bf-cbc'), 'hex'); +ERROR: encrypt error: Key was too big +-- blowfish-448 +SELECT encode(encrypt( +decode('fedcba9876543210', 'hex'), +decode('f0e1d2c3b4a5968778695a4b3c2d1e0f001122334455667704689104c2fd3b2f584023641aba61761f1f1f1f0e0e0e0effffffffffffffff', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Key was too big +-- result: c04504012e4e1f53 +-- empty data +select encode(encrypt('', 'foo', 'bf'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 10 bytes key +select encode(encrypt('foo', '0123456789', 'bf'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 22 bytes key +select encode(encrypt('foo', '0123456789012345678901', 'bf'), 'hex'); +ERROR: encrypt error: Key was too big +-- decrypt +select decrypt(encrypt('foo', '0123456', 'bf'), '0123456', 'bf'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- iv +select encode(encrypt_iv('foo', '0123456', 'abcd', 'bf'), 'hex'); +ERROR: encrypt_iv error: Cipher cannot be initialized ? +select decrypt_iv(decode('95c7e89322525d59', 'hex'), '0123456', 'abcd', 'bf'); +ERROR: decrypt_iv error: Cipher cannot be initialized ? +-- long message +select encode(encrypt('Lets try a longer message.', '0123456789', 'bf'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +select decrypt(encrypt('Lets try a longer message.', '0123456789', 'bf'), '0123456789', 'bf'); +ERROR: encrypt error: Cipher cannot be initialized ? diff --git a/contrib/pgcrypto/expected/cast5_1.out b/contrib/pgcrypto/expected/cast5_1.out new file mode 100644 index 000000000000..e3b38dbce2fe --- /dev/null +++ b/contrib/pgcrypto/expected/cast5_1.out @@ -0,0 +1,48 @@ +-- +-- Cast5 cipher +-- +-- ensure consistent test output regardless of the default bytea format +SET bytea_output TO escape; +-- test vectors from RFC2144 +-- 128 bit key +SELECT encode(encrypt( +decode('01 23 45 67 89 AB CD EF', 'hex'), +decode('01 23 45 67 12 34 56 78 23 45 67 89 34 56 78 9A', 'hex'), +'cast5-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- result: 23 8B 4F E5 84 7E 44 B2 +-- 80 bit key +SELECT encode(encrypt( +decode('01 23 45 67 89 AB CD EF', 'hex'), +decode('01 23 45 67 12 34 56 78 23 45', 'hex'), +'cast5-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- result: EB 6A 71 1A 2C 02 27 1B +-- 40 bit key +SELECT encode(encrypt( +decode('01 23 45 67 89 AB CD EF', 'hex'), +decode('01 23 45 67 12', 'hex'), +'cast5-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- result: 7A C8 16 D1 6E 9B 30 2E +-- cbc +-- empty data +select encode( encrypt('', 'foo', 'cast5'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 10 bytes key +select encode( encrypt('foo', '0123456789', 'cast5'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- decrypt +select decrypt(encrypt('foo', '0123456', 'cast5'), '0123456', 'cast5'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- iv +select encode(encrypt_iv('foo', '0123456', 'abcd', 'cast5'), 'hex'); +ERROR: encrypt_iv error: Cipher cannot be initialized ? +select decrypt_iv(decode('384a970695ce016a', 'hex'), + '0123456', 'abcd', 'cast5'); +ERROR: decrypt_iv error: Cipher cannot be initialized ? +-- long message +select encode(encrypt('Lets try a longer message.', '0123456789', 'cast5'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +select decrypt(encrypt('Lets try a longer message.', '0123456789', 'cast5'), '0123456789', 'cast5'); +ERROR: encrypt error: Cipher cannot be initialized ? diff --git a/contrib/pgcrypto/expected/des_1.out b/contrib/pgcrypto/expected/des_1.out new file mode 100644 index 000000000000..e8cca0505fe1 --- /dev/null +++ b/contrib/pgcrypto/expected/des_1.out @@ -0,0 +1,31 @@ +-- +-- DES cipher +-- +-- ensure consistent test output regardless of the default bytea format +SET bytea_output TO escape; +-- no official test vectors atm +-- from blowfish.sql +SELECT encode(encrypt( +decode('0123456789abcdef', 'hex'), +decode('fedcba9876543210', 'hex'), +'des-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- empty data +select encode( encrypt('', 'foo', 'des'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 8 bytes key +select encode( encrypt('foo', '01234589', 'des'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- decrypt +select decrypt(encrypt('foo', '0123456', 'des'), '0123456', 'des'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- iv +select encode(encrypt_iv('foo', '0123456', 'abcd', 'des'), 'hex'); +ERROR: encrypt_iv error: Cipher cannot be initialized ? +select decrypt_iv(decode('50735067b073bb93', 'hex'), '0123456', 'abcd', 'des'); +ERROR: decrypt_iv error: Cipher cannot be initialized ? +-- long message +select encode(encrypt('Lets try a longer message.', '01234567', 'des'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +select decrypt(encrypt('Lets try a longer message.', '01234567', 'des'), '01234567', 'des'); +ERROR: encrypt error: Cipher cannot be initialized ? diff --git a/contrib/pgcrypto/expected/pgp-decrypt_1.out b/contrib/pgcrypto/expected/pgp-decrypt_1.out new file mode 100644 index 000000000000..63d5ab98654f --- /dev/null +++ b/contrib/pgcrypto/expected/pgp-decrypt_1.out @@ -0,0 +1,421 @@ +-- +-- pgp decrypt tests +-- +-- Checking ciphers +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.blowfish.sha1.mdc.s2k3.z0 + +jA0EBAMCfFNwxnvodX9g0jwB4n4s26/g5VmKzVab1bX1SmwY7gvgvlWdF3jKisvS +yA6Ce1QTMK3KdL2MPfamsTUSAML8huCJMwYQFfE= +=JcP+ +-----END PGP MESSAGE----- +'), 'foobar'); +ERROR: Wrong key or corrupt data +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCci97v0Q6Z0Zg0kQBsVf5Oe3iC+FBzUmuMV9KxmAyOMyjCc/5i8f1Eest +UTAsG35A1vYs02VARKzGz6xI2UHwFUirP+brPBg3Ee7muOx8pA== +=XtrP +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k3.z0 + +jA0ECAMCI7YQpWqp3D1g0kQBCjB7GlX7+SQeXNleXeXQ78ZAPNliquGDq9u378zI +5FPTqAhIB2/2fjY8QEIs1ai00qphjX2NitxV/3Wn+6dufB4Q4g== +=rCZt +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k3.z0 + +jA0ECQMC4f/5djqCC1Rg0kQBTHEPsD+Sw7biBsM2er3vKyGPAQkuTBGKC5ie7hT/ +lceMfQdbAg6oTFyJpk/wH18GzRDphCofg0X8uLgkAKMrpcmgog== +=fB6S +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking MDC modes +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.nomdc.s2k3.z0 + +jA0EBwMCnv07rlXqWctgyS2Dm2JfOKCRL4sLSLJUC8RS2cH7cIhKSuLitOtyquB+ +u9YkgfJfsuRJmgQ9tmo= +=60ui +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCEeP3idNjQ1Bg0kQBf4G0wX+2QNzLh2YNwYkQgQkfYhn/hLXjV4nK9nsE +8Ex1Dsdt5UPvOz8W8VKQRS6loOfOe+yyXil8W3IYFwUpdDUi+Q== +=moGf +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking hashes +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.md5.mdc.s2k3.z0 + +jA0EBwMClrXXtOXetohg0kQBn0Kl1ymevQZRHkdoYRHgzCwSQEiss7zYff2UNzgO +KyRrHf7zEBuZiZ2AG34jNVMOLToj1jJUg5zTSdecUzQVCykWTA== +=NyLk +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCApbdlrURoWJg0kQBzHM/E0o7djY82bNuspjxjAcPFrrtp0uvDdMQ4z2m +/PM8jhgI5vxFYfNQjLl8y3fHYIomk9YflN9K/Q13iq8A8sjeTw== +=FxbQ +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking S2K modes +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k0.z0 + +jAQEBwAC0kQBKTaLAKE3xzps+QIZowqRNb2eAdzBw2LxEW2YD5PgNlbhJdGg+dvw +Ah9GXjGS1TVALzTImJbz1uHUZRfhJlFbc5yGQw== +=YvkV +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k1.z0 + +jAwEBwEC/QTByBLI3b/SRAHPxKzI6SZBo5lAEOD+EsvKQWO4adL9tDY+++Iqy1xK +4IaWXVKEj9R2Lr2xntWWMGZtcKtjD2lFFRXXd9dZp1ZThNDz +=dbXm +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCEq4Su3ZqNEJg0kQB4QG5jBTKF0i04xtH+avzmLhstBNRxvV3nsmB3cwl +z+9ZaA/XdSx5ZiFnMym8P6r8uY9rLjjNptvvRHlxIReF+p9MNg== +=VJKg +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k0.z0 + +jAQECAAC0kQBBDnQWkgsx9YFaqDfWmpsiyAJ6y2xG/sBvap1dySYEMuZ+wJTXQ9E +Cr3i2M7TgVZ0M4jp4QL0adG1lpN5iK7aQeOwMw== +=cg+i +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k1.z0 + +jAwECAECruOfyNDFiTnSRAEVoGXm4A9UZKkWljdzjEO/iaE7mIraltIpQMkiqCh9 +7h8uZ2u9uRBOv222fZodGvc6bvq/4R4hAa/6qSHtm8mdmvGt +=aHmC +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k3.z0 + +jA0ECAMCjFn6SRi3SONg0kQBqtSHPaD0m7rXfDAhCWU/ypAsI93GuHGRyM99cvMv +q6eF6859ZVnli3BFSDSk3a4e/pXhglxmDYCfjAXkozKNYLo6yw== +=K0LS +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k0.z0 + +jAQECQAC0kQB4L1eMbani07XF2ZYiXNK9LW3v8w41oUPl7dStmrJPQFwsdxmrDHu +rQr3WbdKdY9ufjOE5+mXI+EFkSPrF9rL9NCq6w== +=RGts +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k1.z0 + +jAwECQECKHhrou7ZOIXSRAHWIVP+xjVQcjAVBTt+qh9SNzYe248xFTwozkwev3mO ++KVJW0qhk0An+Y2KF99/bYFl9cL5D3Tl43fC8fXGl3x3m7pR +=SUrU +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k3.z0 + +jA0ECQMCjc8lwZu8Fz1g0kQBkEzjImi21liep5jj+3dAJ2aZFfUkohi8b3n9z+7+ +4+NRzL7cMW2RLAFnJbiqXDlRHMwleeuLN1up2WIxsxtYYuaBjA== +=XZrG +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking longer passwords +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCx6dBiuqrYNRg0kQBEo63AvA1SCslxP7ayanLf1H0/hlk2nONVhTwVEWi +tTGup1mMz6Cfh1uDRErUuXpx9A0gdMu7zX0o5XjrL7WGDAZdSw== +=XKKG +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCBDvYuS990iFg0kQBW31UK5OiCjWf5x6KJ8qNNT2HZWQCjCBZMU0XsOC6 +CMxFKadf144H/vpoV9GA0f22keQgCl0EsTE4V4lweVOPTKCMJg== +=gWDh +-----END PGP MESSAGE----- +'), '0123456789abcdefghij2jk4h5g2j54khg23h54g2kh54g2khj54g23hj54'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCqXbFafC+ofVg0kQBejyiPqH0QMERVGfmPOjtAxvyG5KDIJPYojTgVSDt +FwsDabdQUz5O7bgNSnxfmyw1OifGF+W2bIn/8W+0rDf8u3+O+Q== +=OxOF +-----END PGP MESSAGE----- +'), 'x'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking various data +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCGJ+SpuOysINg0kQBJfSjzsW0x4OVcAyr17O7FBvMTwIGeGcJd99oTQU8 +Xtx3kDqnhUq9Z1fS3qPbi5iNP2A9NxOBxPWz2JzxhydANlgbxg== +=W/ik +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'), 'sha1'), 'hex'); + encode +------------------------------------------ + 0225e3ede6f2587b076d021a189ff60aad67e066 +(1 row) + +-- expected: 0225e3ede6f2587b076d021a189ff60aad67e066 +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat2.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCvdpDvidNzMxg0jUBvj8eS2+1t/9/zgemxvhtc0fvdKGGbjH7dleaTJRB +SaV9L04ky1qECNDx3XjnoKLC+H7IOQ== +=Fxen +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'), 'sha1'), 'hex'); + encode +------------------------------------------ + da39a3ee5e6b4b0d3255bfef95601890afd80709 +(1 row) + +-- expected: da39a3ee5e6b4b0d3255bfef95601890afd80709 +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat3.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCxQvxJZ3G/HRg0lgBeYmTa7/uDAjPyFwSX4CYBgpZWVn/JS8JzILrcWF8 +gFnkUKIE0PSaYFp+Yi1VlRfUtRQ/X/LYNGa7tWZS+4VQajz2Xtz4vUeAEiYFYPXk +73Hb8m1yRhQK +=ivrD +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'), 'sha1'), 'hex'); + encode +------------------------------------------ + 5e5c135efc0dd00633efc6dfd6e731ea408a5b4c +(1 row) + +-- expected: 5e5c135efc0dd00633efc6dfd6e731ea408a5b4c +-- Checking CRLF +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: crlf mess + +ww0ECQMCt7VAtby6l4Bi0lgB5KMIZiiF/b3CfMfUyY0eDncsGXtkbu1X+l9brjpMP8eJnY79Amms +a3nsOzKTXUfS9VyaXo8IrncM6n7fdaXpwba/3tNsAhJG4lDv1k4g9v8Ix2dfv6Rs +=mBP9 +-----END PGP MESSAGE----- +'), 'key', 'convert-crlf=0'), 'sha1'), 'hex'); + encode +------------------------------------------ + 9353062be7720f1446d30b9e75573a4833886784 +(1 row) + +-- expected: 9353062be7720f1446d30b9e75573a4833886784 +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: crlf mess + +ww0ECQMCt7VAtby6l4Bi0lgB5KMIZiiF/b3CfMfUyY0eDncsGXtkbu1X+l9brjpMP8eJnY79Amms +a3nsOzKTXUfS9VyaXo8IrncM6n7fdaXpwba/3tNsAhJG4lDv1k4g9v8Ix2dfv6Rs +=mBP9 +-----END PGP MESSAGE----- +'), 'key', 'convert-crlf=1'), 'sha1'), 'hex'); + encode +------------------------------------------ + 7efefcab38467f7484d6fa43dc86cf5281bd78e2 +(1 row) + +-- expected: 7efefcab38467f7484d6fa43dc86cf5281bd78e2 +-- check BUG #11905, problem with messages 6 less than a power of 2. +select pgp_sym_decrypt(pgp_sym_encrypt(repeat('x',65530),'1'),'1') = repeat('x',65530); + ?column? +---------- + t +(1 row) + +-- expected: true +-- Negative tests +-- Decryption with a certain incorrect key yields an apparent Literal Data +-- packet reporting its content to be binary data. Ciphertext source: +-- iterative pgp_sym_encrypt('secret', 'key') until the random prefix gave +-- rise to that property. +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +ww0EBwMCxf8PTrQBmJdl0jcB6y2joE7GSLKRv7trbNsF5Z8ou5NISLUg31llVH/S0B2wl4bvzZjV +VsxxqLSPzNLAeIspJk5G +=mSd/ +-----END PGP MESSAGE----- +'), 'wrong-key', 'debug=1'); +NOTICE: dbg: prefix_init: corrupt prefix +NOTICE: dbg: parse_literal_data: data type=b +NOTICE: dbg: mdcbuf_finish: bad MDC pkt hdr +ERROR: Wrong key or corrupt data +-- Routine text/binary mismatch. +select pgp_sym_decrypt(pgp_sym_encrypt_bytea('P', 'key'), 'key', 'debug=1'); +NOTICE: dbg: parse_literal_data: data type=b +ERROR: Not text data +-- Decryption with a certain incorrect key yields an apparent BZip2-compressed +-- plaintext. Ciphertext source: iterative pgp_sym_encrypt('secret', 'key') +-- until the random prefix gave rise to that property. +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +ww0EBwMC9rK/dMkF5Zlt0jcBlzAQ1mQY2qYbKYbw8h3EZ5Jk0K2IiY92R82TRhWzBIF/8cmXDPtP +GXsd65oYJZp3Khz0qfyn +=Nmpq +-----END PGP MESSAGE----- +'), 'wrong-key', 'debug=1'); +NOTICE: dbg: prefix_init: corrupt prefix +NOTICE: dbg: parse_compressed_data: bzip2 unsupported +NOTICE: dbg: mdcbuf_finish: bad MDC pkt hdr +ERROR: Wrong key or corrupt data +-- Routine use of BZip2 compression. Ciphertext source: +-- echo x | gpg --homedir /nonexistent --personal-compress-preferences bzip2 \ +-- --personal-cipher-preferences aes --no-emit-version --batch \ +-- --symmetric --passphrase key --armor +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +jA0EBwMCRhFrAKNcLVJg0mMBLJG1cCASNk/x/3dt1zJ+2eo7jHfjgg3N6wpB3XIe +QCwkWJwlBG5pzbO5gu7xuPQN+TbPJ7aQ2sLx3bAHhtYb0i3vV9RO10Gw++yUyd4R +UCAAw2JRIISttRHMfDpDuZJpvYo= +=AZ9M +-----END PGP MESSAGE----- +'), 'key', 'debug=1'); +NOTICE: dbg: parse_compressed_data: bzip2 unsupported +ERROR: Unsupported compression algorithm diff --git a/contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out b/contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out new file mode 100644 index 000000000000..f41c6c9893ab --- /dev/null +++ b/contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out @@ -0,0 +1,652 @@ +-- +-- PGP Public Key Encryption +-- +-- As most of the low-level stuff is tested in symmetric key +-- tests, here's only public-key specific tests +create table keytbl ( + id int4, + name text, + pubkey text, + seckey text +); +create table encdata ( + id int4, + data text +); +insert into keytbl (id, name, pubkey, seckey) +values (1, 'elg1024', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQLQfRWxnYW1hbCAx +MDI0IDx0ZXN0QGV4YW1wbGUub3JnPoheBBMRAgAeBQJCyCFIAhsDBgsJCAcDAgMV +AgMDFgIBAh4BAheAAAoJEBwpvA0YF3NkOtsAniI9W2bC3CxARTpYrev7ihreDzFc +AJ9WYLQxDQAi5Ec9AQoodPkIagzZ4LkBDQRCyCFKEAQAh5SNbbJMAsJ+sQbcWEzd +ku8AdYB5zY7Qyf9EOvn0g39bzANhxmmb6gbRlQN0ioymlDwraTKUAfuCZgNcg/0P +sxFGb9nDcvjIV8qdVpnq1PuzMFuBbmGI6weg7Pj01dlPiO0wt1lLX+SubktqbYxI ++h31c3RDZqxj+KAgxR8YNGMAAwYD+wQs2He1Z5+p4OSgMERiNzF0acZUYmc0e+/9 +6gfL0ft3IP+SSFo6hEBrkKVhZKoPSSRr5KpNaEobhdxsnKjUaw/qyoaFcNMzb4sF +k8wq5UlCkR+h72u6hv8FuleCV8SJUT1U2JjtlXJR2Pey9ifh8rZfu57UbdwdHa0v +iWc4DilhiEkEGBECAAkFAkLIIUoCGwwACgkQHCm8DRgXc2TtrwCfdPom+HlNVE9F +ig3hGY1Rb4NEk1gAn1u9IuQB+BgDP40YHHz6bKWS/x80 +=RWci +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQG7BELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQAAAnj4i4st+s+C6 +WKTIDcL1Iy0Saq8lCp60H0VsZ2FtYWwgMTAyNCA8dGVzdEBleGFtcGxlLm9yZz6I +XgQTEQIAHgUCQsghSAIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRAcKbwNGBdz +ZDrbAJ9cp6AsjOhiLxwznsMJheGf4xkH8wCfUPjMCLm4tAEnyYn2hDNt7CB8B6Kd +ATEEQsghShAEAIeUjW2yTALCfrEG3FhM3ZLvAHWAec2O0Mn/RDr59IN/W8wDYcZp +m+oG0ZUDdIqMppQ8K2kylAH7gmYDXIP9D7MRRm/Zw3L4yFfKnVaZ6tT7szBbgW5h +iOsHoOz49NXZT4jtMLdZS1/krm5Lam2MSPod9XN0Q2asY/igIMUfGDRjAAMGA/sE +LNh3tWefqeDkoDBEYjcxdGnGVGJnNHvv/eoHy9H7dyD/kkhaOoRAa5ClYWSqD0kk +a+SqTWhKG4XcbJyo1GsP6sqGhXDTM2+LBZPMKuVJQpEfoe9ruob/BbpXglfEiVE9 +VNiY7ZVyUdj3svYn4fK2X7ue1G3cHR2tL4lnOA4pYQAA9030E4u2ZKOfJBpUM+EM +m9VmsGjaQZV4teB0R/q3W8sRIYhJBBgRAgAJBQJCyCFKAhsMAAoJEBwpvA0YF3Nk +7a8AniFFotw1x2X+oryu3Q3nNtmxoKHpAJ9HU7jw7ydg33dI9J8gVkrmsSZ2/w== +=nvqq +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (2, 'elg2048', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELIIgoRBAC1onBpxKYgDvrgCaUWPY34947X3ogxGOfCN0p6Eqrx+2PUhm4n +vFvmczpMT4iDc0mUO+iwnwsEkXQI1eC99g8c0jnZAvzJZ5miAHL8hukMAMfDkYke +5aVvcPPc8uPDlItpszGmH0rM0V9TIt/i9QEXetpyNWhk4jj5qnohYhLeZwCgkOdO +RFAdNi4vfFPivvtAp2ffjU8D/R3x/UJCvkzi7i9rQHGo313xxmQu5BuqIjANBUij +8IE7LRPI/Qhg2hYy3sTJwImDi7VkS+fuvNVk0d6MTWplAXYU96bn12JaD21R9sKl +Fzcc+0iZI1wYA1PczisUkoTISE+dQFUsoGHfpDLhoBuesXQrhBavI8t8VPd+nkdt +J+oKA/9iRQ87FzxdYTkh2drrv69FZHc3Frsjw9nPcBq/voAvXH0MRilqyCg7HpW/ +T9naeOERksa+Rj4R57IF1l4e5oiiGJo9QmaKZcsCsXrREJCycrlEtMqXfSPy+bi5 +0yDZE/Qm1dwu13+OXOsRvkoNYjO8Mzo9K8wU12hMqN0a2bu6a7QjRWxnYW1hbCAy +MDQ4IDx0ZXN0MjA0OEBleGFtcGxlLm9yZz6IXgQTEQIAHgUCQsgiCgIbAwYLCQgH +AwIDFQIDAxYCAQIeAQIXgAAKCRBI6c1W/qZo29PDAKCG724enIxRog1j+aeCp/uq +or6mbwCePuKy2/1kD1FvnhkZ/R5fpm+pdm25Ag0EQsgiIhAIAJI3Gb2Ehtz1taQ9 +AhPY4Avad2BsqD3S5X/R11Cm0KBE/04D29dxn3f8QfxDsexYvNIZjoJPBqqZ7iMX +MhoWyw8ZF5Zs1mLIjFGVorePrm94N3MNPWM7x9M36bHUjx0vCZKFIhcGY1g+htE/ +QweaJzNVeA5z4qZmik41FbQyQSyHa3bOkTZu++/U6ghP+iDp5UDBjMTkVyqITUVN +gC+MR+da/I60irBVhue7younh4ovF+CrVDQJC06HZl6CAJJyA81SmRfi+dmKbbjZ +LF6rhz0norPjISJvkIqvdtM4VPBKI5wpgwCzpEqjuiKrAVujRT68zvBvJ4aVqb11 +k5QdJscAAwUH/jVJh0HbWAoiFTe+NvohfrA8vPcD0rtU3Y+siiqrabotnxJd2NuC +bxghJYGfNtnx0KDjFbCRKJVeTFok4UnuVYhXdH/c6i0/rCTNdeW2D6pmR4GfBozR +Pw/ARf+jONawGLyUj7uq13iquwMSE7VyNuF3ycL2OxXjgOWMjkH8c+zfHHpjaZ0R +QsetMq/iNBWraayKZnWUd+eQqNzE+NUo7w1jAu7oDpy+8a1eipxzK+O0HfU5LTiF +Z1Oe4Um0P2l3Xtx8nEgj4vSeoEkl2qunfGW00ZMMTCWabg0ZgxPzMfMeIcm6525A +Yn2qL+X/qBJTInAl7/hgPz2D1Yd7d5/RdWaISQQYEQIACQUCQsgiIgIbDAAKCRBI +6c1W/qZo25ZSAJ98WTrtl2HiX8ZqZq95v1+9cHtZPQCfZDoWQPybkNescLmXC7q5 +1kNTmEU= +=8QM5 +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQG7BELIIgoRBAC1onBpxKYgDvrgCaUWPY34947X3ogxGOfCN0p6Eqrx+2PUhm4n +vFvmczpMT4iDc0mUO+iwnwsEkXQI1eC99g8c0jnZAvzJZ5miAHL8hukMAMfDkYke +5aVvcPPc8uPDlItpszGmH0rM0V9TIt/i9QEXetpyNWhk4jj5qnohYhLeZwCgkOdO +RFAdNi4vfFPivvtAp2ffjU8D/R3x/UJCvkzi7i9rQHGo313xxmQu5BuqIjANBUij +8IE7LRPI/Qhg2hYy3sTJwImDi7VkS+fuvNVk0d6MTWplAXYU96bn12JaD21R9sKl +Fzcc+0iZI1wYA1PczisUkoTISE+dQFUsoGHfpDLhoBuesXQrhBavI8t8VPd+nkdt +J+oKA/9iRQ87FzxdYTkh2drrv69FZHc3Frsjw9nPcBq/voAvXH0MRilqyCg7HpW/ +T9naeOERksa+Rj4R57IF1l4e5oiiGJo9QmaKZcsCsXrREJCycrlEtMqXfSPy+bi5 +0yDZE/Qm1dwu13+OXOsRvkoNYjO8Mzo9K8wU12hMqN0a2bu6awAAn2F+iNBElfJS +8azqO/kEiIfpqu6/DQG0I0VsZ2FtYWwgMjA0OCA8dGVzdDIwNDhAZXhhbXBsZS5v +cmc+iF0EExECAB4FAkLIIgoCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQSOnN +Vv6maNvTwwCYkpcJmpl3aHCQdGomz7dFohDgjgCgiThZt2xTEi6GhBB1vuhk+f55 +n3+dAj0EQsgiIhAIAJI3Gb2Ehtz1taQ9AhPY4Avad2BsqD3S5X/R11Cm0KBE/04D +29dxn3f8QfxDsexYvNIZjoJPBqqZ7iMXMhoWyw8ZF5Zs1mLIjFGVorePrm94N3MN +PWM7x9M36bHUjx0vCZKFIhcGY1g+htE/QweaJzNVeA5z4qZmik41FbQyQSyHa3bO +kTZu++/U6ghP+iDp5UDBjMTkVyqITUVNgC+MR+da/I60irBVhue7younh4ovF+Cr +VDQJC06HZl6CAJJyA81SmRfi+dmKbbjZLF6rhz0norPjISJvkIqvdtM4VPBKI5wp +gwCzpEqjuiKrAVujRT68zvBvJ4aVqb11k5QdJscAAwUH/jVJh0HbWAoiFTe+Nvoh +frA8vPcD0rtU3Y+siiqrabotnxJd2NuCbxghJYGfNtnx0KDjFbCRKJVeTFok4Unu +VYhXdH/c6i0/rCTNdeW2D6pmR4GfBozRPw/ARf+jONawGLyUj7uq13iquwMSE7Vy +NuF3ycL2OxXjgOWMjkH8c+zfHHpjaZ0RQsetMq/iNBWraayKZnWUd+eQqNzE+NUo +7w1jAu7oDpy+8a1eipxzK+O0HfU5LTiFZ1Oe4Um0P2l3Xtx8nEgj4vSeoEkl2qun +fGW00ZMMTCWabg0ZgxPzMfMeIcm6525AYn2qL+X/qBJTInAl7/hgPz2D1Yd7d5/R +dWYAAVQKFPXbRaxbdArwRVXMzSD3qj/+VwwhwEDt8zmBGnlBfwVdkjQQrDUMmV1S +EwyISQQYEQIACQUCQsgiIgIbDAAKCRBI6c1W/qZo25ZSAJ4sgUfHTVsG/x3p3fcM +3b5R86qKEACggYKSwPWCs0YVRHOWqZY0pnHtLH8= +=3Dgk +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (3, 'elg4096', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELII7wRBACFuaAvb11cIvjJK9LkZr4cYuYhLWh3DJdojNNnLNiym5OEksvY +05cw8OgqKtPzICU7o/mHXTWhzJYUt3i50/AeYygI8Q0uATS6RnDAKNlES1EMoHKz +2a5iFbYs4bm4IwlkvYd8uWjcu+U0YLbxir39u+anIc6eT+q3WiH/q3zDRwCgkT98 +cnIG8iO8PdwDSP8G4Lt6TYED/R45GvCzJ4onQALLE92KkLUz8aFWSl05r84kczEN +SxiP9Ss6m465RmwWHfwYAu4b+c4GeNyU8fIU2EM8cezchC+edEi3xu1s+pCV0Dk4 +18DGC8WKCICO30vBynuNmYg7W/7Zd4wtjss454fMW7+idVDNM701mmXBtI1nsBtG +7Z4tA/9FxjFbJK9jh24RewfjHpLYqcfCo2SsUjOwsnMZ5yg2yv9KyVVQhRqwmrqt +q8MRyjGmfoD9PPdCgvqgzy0hHvAHUtTm2zUczGTG+0g4hNIklxC/Mv6J4KE+NWTh +uB4acqofHyaw2WnKOuRUsoDi6rG5AyjNMyAK/vVcEGj7J1tk27QjRWxnYW1hbCA0 +MDk2IDx0ZXN0NDA5NkBleGFtcGxlLm9yZz6IXgQTEQIAHgUCQsgjvAIbAwYLCQgH +AwIDFQIDAxYCAQIeAQIXgAAKCRBj+HX2P2d0oAEDAJ9lI+CNmb42z3+a6TnVusM6 +FI7oLwCfUwA1zEcRdsT3nIkoYh0iKxFSDFW5BA0EQsgkdhAQAJQbLXlgcJ/jq+Xh +Eujb77/eeftFJObNIRYD9fmJ7HFIXbUcknEpbs+cRH/nrj5dGSY3OT3jCXOUtvec +sCoX/CpZWL0oqDjAiZtNSFiulw5Gav4gHYkWKgKdSo+2rkavEPqKIVHvMeXaJtGT +d7v/AmL/P8T7gls93o5WFBOLtPbDvWqaKRy2U5TAhl1laiM0vGALRVjvSCgnGw9g +FpSnXbO3AfenUSjDzZujfGLHtU44ixHSS/D4DepiF3YaYLsN4CBqZRv6FbMZD5W3 +DnJY4kS1kH0MzdcF19TlcZ3itTCcGIt1tMKf84mccPoqdMzH7vumBGTeFEly5Afp +9berJcirqh2fzlunN0GS02z6SGWnjTbDlkNDxuxPSBbpcpNyD3jpYAUqSwRsZ/+5 +zkzcbGtDmvy9sJ5lAXkxGoIoQ1tEVX/LOHnh2NQHK8ourVOnr7MS0nozssITZJ5E +XqtHiREjiYEuPyZiVZKJHLWuYYaF+n40znnz3sJuXFRreHhHbbvRdlYUU5mJV+XZ +BLgKuS33NdpGeMIngnCc/9IQ6OZb6ixc94kbkd3w2PVr8CbKlu/IHTjWOO2mAo+D ++OydlYl23FiM3KOyMP1HcEOJMB/nwkMtrvd+522Lu9n77ktKfot9IPrQDIQTyXjR +3pCOFtCOBnk2tJHMPoG9jn9ah/LHAAMHEACDZ5I/MHGfmiKg2hrmqBu2J2j/deC8 +CpwcyDH1ovQ0gHvb9ESa+CVRU2Wdy2CD7Q9SmtMverB5eneL418iPVRcQdwRmQ2y +IH4udlBa6ce9HTUCaecAZ4/tYBnaC0Av/9l9tz14eYcwRMDpB+bnkhgF+PZ1KAfD +9wcY2aHbtsf3lZBc5h4owPJkxpe/BNzuJxW3q4VpSbLsZhwnCZ2wg7DRwP44wFIk +00ptmoBY59gsU6I40XtzrF8JDr0cA57xND5RY21Z8lnnYRE1Tc8h5REps9ZIxW3/ +yl91404bPLqxczpUHQAMSTAmBaStPYX1nS51uofOhLs5SKPCUmxfGKIOhsD0oLUn +78DnkONVGeXzBibSwwtbgfMzee4G8wSUfJ7w8WXz1TyanaGLnJ+DuKASSOrFoBCD +HEDuWZWgSL74NOQupFRk0gxOPmqU94Y8HziQWma/cETbmD83q8rxN+GM2oBxQkQG +xcbqMTHE7aVhV3tymbSWVaYhww3oIwsZS9oUIi1DnPEowS6CpVRrwdvLjLJnJzzV +O3AFPn9eZ1Q7R1tNx+zZ4OOfhvI/OlRJ3HBx2L53embkbdY9gFYCCdTjPyjKoDIx +kALgCajjCYMNUsAKNSd6mMCQ8TtvukSzkZS1RGKP27ohsdnzIVsiEAbxDMMcI4k1 +ul0LExUTCXSjeIhJBBgRAgAJBQJCyCR2AhsMAAoJEGP4dfY/Z3Sg19sAn0NDS8pb +qrMpQAxSb7zRTmcXEFd9AJ435H0ttP/NhLHXC9ezgbCMmpXMOQ== +=kRxT +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQG7BELII7wRBACFuaAvb11cIvjJK9LkZr4cYuYhLWh3DJdojNNnLNiym5OEksvY +05cw8OgqKtPzICU7o/mHXTWhzJYUt3i50/AeYygI8Q0uATS6RnDAKNlES1EMoHKz +2a5iFbYs4bm4IwlkvYd8uWjcu+U0YLbxir39u+anIc6eT+q3WiH/q3zDRwCgkT98 +cnIG8iO8PdwDSP8G4Lt6TYED/R45GvCzJ4onQALLE92KkLUz8aFWSl05r84kczEN +SxiP9Ss6m465RmwWHfwYAu4b+c4GeNyU8fIU2EM8cezchC+edEi3xu1s+pCV0Dk4 +18DGC8WKCICO30vBynuNmYg7W/7Zd4wtjss454fMW7+idVDNM701mmXBtI1nsBtG +7Z4tA/9FxjFbJK9jh24RewfjHpLYqcfCo2SsUjOwsnMZ5yg2yv9KyVVQhRqwmrqt +q8MRyjGmfoD9PPdCgvqgzy0hHvAHUtTm2zUczGTG+0g4hNIklxC/Mv6J4KE+NWTh +uB4acqofHyaw2WnKOuRUsoDi6rG5AyjNMyAK/vVcEGj7J1tk2wAAoJCUNy6awTkw +XfbLbpqh0fvDst7jDLa0I0VsZ2FtYWwgNDA5NiA8dGVzdDQwOTZAZXhhbXBsZS5v +cmc+iF4EExECAB4FAkLII7wCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQY/h1 +9j9ndKABAwCeNEOVK87EzXYbtxYBsnjrUI948NIAn2+f3BXiBFDV5NvqPwIZ0m77 +Fwy4nQRMBELIJHYQEACUGy15YHCf46vl4RLo2++/3nn7RSTmzSEWA/X5iexxSF21 +HJJxKW7PnER/564+XRkmNzk94wlzlLb3nLAqF/wqWVi9KKg4wImbTUhYrpcORmr+ +IB2JFioCnUqPtq5GrxD6iiFR7zHl2ibRk3e7/wJi/z/E+4JbPd6OVhQTi7T2w71q +mikctlOUwIZdZWojNLxgC0VY70goJxsPYBaUp12ztwH3p1Eow82bo3xix7VOOIsR +0kvw+A3qYhd2GmC7DeAgamUb+hWzGQ+Vtw5yWOJEtZB9DM3XBdfU5XGd4rUwnBiL +dbTCn/OJnHD6KnTMx+77pgRk3hRJcuQH6fW3qyXIq6odn85bpzdBktNs+khlp402 +w5ZDQ8bsT0gW6XKTcg946WAFKksEbGf/uc5M3GxrQ5r8vbCeZQF5MRqCKENbRFV/ +yzh54djUByvKLq1Tp6+zEtJ6M7LCE2SeRF6rR4kRI4mBLj8mYlWSiRy1rmGGhfp+ +NM55897CblxUa3h4R2270XZWFFOZiVfl2QS4Crkt9zXaRnjCJ4JwnP/SEOjmW+os +XPeJG5Hd8Nj1a/AmypbvyB041jjtpgKPg/jsnZWJdtxYjNyjsjD9R3BDiTAf58JD +La73fudti7vZ++5LSn6LfSD60AyEE8l40d6QjhbQjgZ5NrSRzD6BvY5/WofyxwAD +BxAAg2eSPzBxn5oioNoa5qgbtido/3XgvAqcHMgx9aL0NIB72/REmvglUVNlnctg +g+0PUprTL3qweXp3i+NfIj1UXEHcEZkNsiB+LnZQWunHvR01AmnnAGeP7WAZ2gtA +L//Zfbc9eHmHMETA6Qfm55IYBfj2dSgHw/cHGNmh27bH95WQXOYeKMDyZMaXvwTc +7icVt6uFaUmy7GYcJwmdsIOw0cD+OMBSJNNKbZqAWOfYLFOiONF7c6xfCQ69HAOe +8TQ+UWNtWfJZ52ERNU3PIeURKbPWSMVt/8pfdeNOGzy6sXM6VB0ADEkwJgWkrT2F +9Z0udbqHzoS7OUijwlJsXxiiDobA9KC1J+/A55DjVRnl8wYm0sMLW4HzM3nuBvME +lHye8PFl89U8mp2hi5yfg7igEkjqxaAQgxxA7lmVoEi++DTkLqRUZNIMTj5qlPeG +PB84kFpmv3BE25g/N6vK8TfhjNqAcUJEBsXG6jExxO2lYVd7cpm0llWmIcMN6CML +GUvaFCItQ5zxKMEugqVUa8Hby4yyZyc81TtwBT5/XmdUO0dbTcfs2eDjn4byPzpU +Sdxwcdi+d3pm5G3WPYBWAgnU4z8oyqAyMZAC4Amo4wmDDVLACjUnepjAkPE7b7pE +s5GUtURij9u6IbHZ8yFbIhAG8QzDHCOJNbpdCxMVEwl0o3gAAckBdfKuasiNUn5G +L5XRnSvaOFzftr8zteOlZChCSNvzH5k+i1j7RJbWq06OeKRywPzjfjgM2MvRzI43 +ICeISQQYEQIACQUCQsgkdgIbDAAKCRBj+HX2P2d0oNfbAJ9+G3SeXrk+dWwo9EGi +hqMi2GVTsgCfeoQJPsc8FLYUgfymc/3xqAVLUtg= +=Gjq6 +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (4, 'rsa2048', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQELBELIJbEBCADAIdtcoLAmQfl8pb73pPRuEYx8qW9klLfCGG5A4OUOi00JHNwP +ZaABe1PGzjoeXrgM1MTQZhoZu1Vdg+KDI6XAtiy9P6bLg7ntsXksD4wBoIKtQKc2 +55pdukxTiu+xeJJG2q8ZZPOp97CV9fbQ9vPCwgnuSsDCoQlibZikDVPAyVTvp7Jx +5rz8yXsl4sxvaeMZPqqFPtA/ENeQ3cpsyR1BQXSvoZpH1Fq0b8GcZTEdWWD/w6/K +MCRC8TmgEd+z3e8kIsCwFQ+TSHbCcxRWdgZE7gE31sJHHVkrZlXtLU8MPXWqslVz +R0cX+yC8j6bXI6/BqZ2SvRndJwuunRAr4um7AAYptB5SU0EgMjA0OCA8cnNhMjA0 +OEBleGFtcGxlLm9yZz6JATQEEwECAB4FAkLIJbECGwMGCwkIBwMCAxUCAwMWAgEC +HgECF4AACgkQnc+OnJvTHyQqHwf8DtzuAGmObfe3ggtn14x2wnU1Nigebe1K5liR +nrLuVlLBpdO6CWmMUzfKRvyZlx54GlA9uUQSjW+RlgejdOTQqesDrcTEukYd4yzw +bLZyM5Gb3lsE/FEmE7Dxw/0Utf59uACqzG8LACQn9J6sEgZWKxAupuYTHXd12lDP +D3dnU4uzKPhMcjnSN00pzjusP7C9NZd3OLkAx2vw/dmb4Q+/QxeZhVYYsAUuR2hv +9bgGWopumlOkt8Zu5YG6+CtTbJXprPI7pJ1jHbeE+q/29hWJQtS8Abx82AcOkzhv +S3NZKoJ/1DrGgoDAu1mGkM4KvLAxfDs/qQ9dZhtEmDbKPLTVEA== +=lR4n +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQOWBELIJbEBCADAIdtcoLAmQfl8pb73pPRuEYx8qW9klLfCGG5A4OUOi00JHNwP +ZaABe1PGzjoeXrgM1MTQZhoZu1Vdg+KDI6XAtiy9P6bLg7ntsXksD4wBoIKtQKc2 +55pdukxTiu+xeJJG2q8ZZPOp97CV9fbQ9vPCwgnuSsDCoQlibZikDVPAyVTvp7Jx +5rz8yXsl4sxvaeMZPqqFPtA/ENeQ3cpsyR1BQXSvoZpH1Fq0b8GcZTEdWWD/w6/K +MCRC8TmgEd+z3e8kIsCwFQ+TSHbCcxRWdgZE7gE31sJHHVkrZlXtLU8MPXWqslVz +R0cX+yC8j6bXI6/BqZ2SvRndJwuunRAr4um7AAYpAAf/QZsrrz0c7dgWwGqMIpw6 +fP+/lLa74+fa2CFRWtYowEiKsfDg/wN7Ua07036dNhPa8aZPsU6SRzm5PybKOURe +D9pNt0FxJkX0j5pCWfjSJgTbc1rCdqZ/oyBk/U6pQtf//zfw3PbDl7I8TC6GOt2w +5NgcXdsWHP7LAmPctOVUyzFsenevR0MFTHkMbmKI1HpFm8XN/e1Fl+qIAD+OagTF +5B32VvpoJtkh5nxnIuToNJsa9Iy7F9MM2CeFOyTMihMcjXKBBUaAYoF115irBvqu +7N/qWmzqLg8yxBZ56mh6meCF3+67VA2y7fL8rhw2QuqgLg1JFlKAVL+9crCSrn// +GQQA1kT7FytW6BNOffblFYZkrJer3icoRDqa/ljgH/yVaWoVT1igy0E9XzYO7MwP +2usj/resLy0NC1qCthk51cZ/wthooMl88e5Wb4l5FYwBEac7muSBTo4W8cAH1hFj +TWL6XAGvEzGX3Mt9pn8uYGlQLZAhJoNCAU2EOCbN1PchDvsEAOWNKYesuUVk8+sQ +St0NDNhd9BWtTWTHkCZb1dKC3JTfr9PqkTBLrWFbYjkOtvdPAW7FDaXXXZfdH1jH +WfwP3Q+I6sqgSaWpCS4dBAns3/RVtO7czVgyIwma04iIvJqderYrfvkUq95KfwP2 +V8wXkhrPPPxyrg5y3wQlpY2jb5RBBAC17SK1ms+DBtck4vpdjp3SJ32SbyC/DU30 +89Q12j74S7Zdu1qZlKnvy3kWPYX/hMuSzGZ+mLVJNFEqH2X01aFzppYz0hdI9PGB +9tTFEqZWQL9ZkXfjc79Cgnt12pNukRbtw0N/kyutOdIFHVT79wVAd+powqziXJsC +Kc+4xjwSCkZitB5SU0EgMjA0OCA8cnNhMjA0OEBleGFtcGxlLm9yZz6JATQEEwEC +AB4FAkLIJbECGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQnc+OnJvTHyQqHwf8 +DtzuAGmObfe3ggtn14x2wnU1Nigebe1K5liRnrLuVlLBpdO6CWmMUzfKRvyZlx54 +GlA9uUQSjW+RlgejdOTQqesDrcTEukYd4yzwbLZyM5Gb3lsE/FEmE7Dxw/0Utf59 +uACqzG8LACQn9J6sEgZWKxAupuYTHXd12lDPD3dnU4uzKPhMcjnSN00pzjusP7C9 +NZd3OLkAx2vw/dmb4Q+/QxeZhVYYsAUuR2hv9bgGWopumlOkt8Zu5YG6+CtTbJXp +rPI7pJ1jHbeE+q/29hWJQtS8Abx82AcOkzhvS3NZKoJ/1DrGgoDAu1mGkM4KvLAx +fDs/qQ9dZhtEmDbKPLTVEA== +=WKAv +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (5, 'psw-elg1024', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQLQfRWxnYW1hbCAx +MDI0IDx0ZXN0QGV4YW1wbGUub3JnPoheBBMRAgAeBQJCyCFIAhsDBgsJCAcDAgMV +AgMDFgIBAh4BAheAAAoJEBwpvA0YF3NkOtsAniI9W2bC3CxARTpYrev7ihreDzFc +AJ9WYLQxDQAi5Ec9AQoodPkIagzZ4LkBDQRCyCFKEAQAh5SNbbJMAsJ+sQbcWEzd +ku8AdYB5zY7Qyf9EOvn0g39bzANhxmmb6gbRlQN0ioymlDwraTKUAfuCZgNcg/0P +sxFGb9nDcvjIV8qdVpnq1PuzMFuBbmGI6weg7Pj01dlPiO0wt1lLX+SubktqbYxI ++h31c3RDZqxj+KAgxR8YNGMAAwYD+wQs2He1Z5+p4OSgMERiNzF0acZUYmc0e+/9 +6gfL0ft3IP+SSFo6hEBrkKVhZKoPSSRr5KpNaEobhdxsnKjUaw/qyoaFcNMzb4sF +k8wq5UlCkR+h72u6hv8FuleCV8SJUT1U2JjtlXJR2Pey9ifh8rZfu57UbdwdHa0v +iWc4DilhiEkEGBECAAkFAkLIIUoCGwwACgkQHCm8DRgXc2TtrwCfdPom+HlNVE9F +ig3hGY1Rb4NEk1gAn1u9IuQB+BgDP40YHHz6bKWS/x80 +=RWci +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQHpBELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQP4HAwImKZ5q2QwT +D2DDAY/IQBjes7WgqZeacfLPDoB8ecD/KLoSCH6Z3etvbPHSOKiazxoJ962Ix74H +ZAE6ZbMTtl5dZW1ptB9FbGdhbWFsIDEwMjQgPHRlc3RAZXhhbXBsZS5vcmc+iF4E +ExECAB4FAkLIIUgCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQHCm8DRgXc2Q6 +2wCfXKegLIzoYi8cM57DCYXhn+MZB/MAn1D4zAi5uLQBJ8mJ9oQzbewgfAeinQFf +BELIIUoQBACHlI1tskwCwn6xBtxYTN2S7wB1gHnNjtDJ/0Q6+fSDf1vMA2HGaZvq +BtGVA3SKjKaUPCtpMpQB+4JmA1yD/Q+zEUZv2cNy+MhXyp1WmerU+7MwW4FuYYjr +B6Ds+PTV2U+I7TC3WUtf5K5uS2ptjEj6HfVzdENmrGP4oCDFHxg0YwADBgP7BCzY +d7Vnn6ng5KAwRGI3MXRpxlRiZzR77/3qB8vR+3cg/5JIWjqEQGuQpWFkqg9JJGvk +qk1oShuF3GycqNRrD+rKhoVw0zNviwWTzCrlSUKRH6Hva7qG/wW6V4JXxIlRPVTY +mO2VclHY97L2J+Hytl+7ntRt3B0drS+JZzgOKWH+BwMCJimeatkMEw9gRkFjt4Xa +9rX8awMBE5+vVcGKv/DNiCvJnlYvSdCj8VfuHsYFliiJo6u17NJon+K43e3yvDNk +f631VOVanGEz7TyqOkWQiEkEGBECAAkFAkLIIUoCGwwACgkQHCm8DRgXc2TtrwCe +IUWi3DXHZf6ivK7dDec22bGgoekAn0dTuPDvJ2Dfd0j0nyBWSuaxJnb/ +=SNvr +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (6, 'rsaenc2048', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQELBELr2m0BCADOrnknlnXI0EzRExf/TgoHvK7Xx/E0keWqV3KrOyC3/tY2KOrj +UVxaAX5pkFX9wdQObGPIJm06u6D16CH6CildX/vxG7YgvvKzK8JGAbwrXAfk7OIW +czO2zRaZGDynoK3mAxHRBReyTKtNv8rDQhuZs6AOozJNARdbyUO/yqUnqNNygWuT +4htFDEuLPIJwAbMSD0BvFW6YQaPdxzaAZm3EWVNbwDzjgbBUdBiUUwRdZIFUhsjJ +dirFdy5+uuZru6y6CNC1OERkJ7P8EyoFiZckAIE5gshVZzNuyLOZjc5DhWBvLbX4 +NZElAnfiv+4nA6y8wQLSIbmHA3nqJaBklj85AAYptCVSU0EgMjA0OCBFbmMgPHJz +YTIwNDhlbmNAZXhhbXBsZS5vcmc+iQE0BBMBAgAeBQJC69ptAhsDBgsJCAcDAgMV +AgMDFgIBAh4BAheAAAoJEMiZ6pNEGVVZHMkIAJtGHHZ9iM8Yq1rr0zl1L6SvlQP8 +JCaxHa31wH3PKqGtq2M+cpb2rXf7gAY/doHJPXggfVzkyFrysmQ1gPbDGYLyOutw ++IkhihEb5bWxQBNj+3zAFs1YX6v2HXWbSUSmyY1V9/+NTtKk03olDc/swd3lXzku +UOhcgfpBgIt3Q+MpT6M2+OIF7lVfSb1rWdpwTfGhZzW9szQOeoS4gPvxCCRyuabQ +RJ6DWH61F8fFIDJg1z+A/Obx4fqX6GOA69RzgZ3oukFBIXxNwV9PZNnAmHtZVYO8 +0g/oVYBbuvOYedffDBeQarhERZ5W2TnIE+nqY61YOLBqosliygdZTXULzNi5AQsE +QuvaugEIAOuCJZdkzORA6e1lr81Lnr4JzMsVBFA+X/yIkBbV6qX/A4nVSLAZKNPX +z1YIrMTu+1rMIiy10IWbA6zgMTpzPhJRfgePONgdnCYyK5Ksh5/C5ntzKwwGwxfK +lAXIxJurCHXTbEa+YvPdn76vJ3HsXOXVEL+fLb4U3l3Ng87YM202Lh1Ha2MeS2zE +FZcAoKbFqAAjDLEai64SoOFh0W3CsD1DL4zmfp+YZrUPHTtZadsi53i4KKW/ws9U +rHlolqYNhYze/uRLyfnUx9PN4r/GhEzauyDMV0smo91uB3aewPft+eCpmeWnu0PF +JVK4xyRmhIq2rVCw16a1pBJirvGM+y0ABimJAR8EGAECAAkFAkLr2roCGwwACgkQ +yJnqk0QZVVku1wgAg1bLSjPkhw+ldG5HzumpqR84+JKyozdJaJzefu2+1iqYE0B0 +WLz2PJVIiK41xiEkKhBvTOQYuXmtWqAWXptD91P5SoXoNJWLQO3TNwarANhHxkWg +w/TOUxQqoctlRUej5NDD+4eW5G9lcS1FEGuKDWtX096u80vO+TbyJjvx2eVM1k+X +dmeYsGOiNgDimCreJGYc14G7eY9jt24gw10n1sMAKI1qm6lcoHqZ9OOyla+wJdro +PYZGO7R8+1O9R22WrK6BYDT5j/1JwMZqbOESjNvDEVT0yOHClCHRN4CChbt6LhKh +CLUNdz/udIt0JAC6c/HdPLSW3HnmM3+iNj+Kug== +=pwU2 +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQOWBELr2m0BCADOrnknlnXI0EzRExf/TgoHvK7Xx/E0keWqV3KrOyC3/tY2KOrj +UVxaAX5pkFX9wdQObGPIJm06u6D16CH6CildX/vxG7YgvvKzK8JGAbwrXAfk7OIW +czO2zRaZGDynoK3mAxHRBReyTKtNv8rDQhuZs6AOozJNARdbyUO/yqUnqNNygWuT +4htFDEuLPIJwAbMSD0BvFW6YQaPdxzaAZm3EWVNbwDzjgbBUdBiUUwRdZIFUhsjJ +dirFdy5+uuZru6y6CNC1OERkJ7P8EyoFiZckAIE5gshVZzNuyLOZjc5DhWBvLbX4 +NZElAnfiv+4nA6y8wQLSIbmHA3nqJaBklj85AAYpAAf9GuKpxrXp267eSPw9ZeSw +Ik6ob1I0MHbhhHeaXQnF0SuOViJ1+Bs74hUB3/F5fqrnjVLIS/ysYzegYpbpXOIa +MZwYcp2e+dpmVb7tkGQgzXH0igGtBQBqoSUVq9mG2XKPVh2JmiYgOH6GrHSGmnCq +GCgEK4ezSomB/3OtPFSjAxOlSw6dXSkapSxW3pEGvCdaWd9p8yl4rSpGsZEErPPL +uSbZZrHtWfgq5UXdPeE1UnMlBcvSruvpN4qgWMgSMs4d2lXvzXJLcht/nryP+atT +H1gwnRmlDCVv5BeJepKo3ORJDvcPlXkJPhqS9If3BhTqt6QgQEFI4aIYYZOZpZoi +2QQA2Zckzktmsc1MS04zS9gm1CbxM9d2KK8EOlh7fycRQhYYqqavhTBH2MgEp+Dd +ZtuEN5saNDe9x/fwi2ok1Bq6luGMWPZU/nZe7fxadzwfliy/qPzStWFW3vY9mMLu +6uEqgjin/lf4YrAswXDZaEc5e4GuNgGfwr27hpjxE1jg3PsEAPMqXEOMT2yh+yRu +DlLRbFhYOI4aUHY2CGoQQONnwv2O5gFvmOcPlg3J5lvnwlOYCx0c3bDxAtHyjPJq +FAZqcJBaB9RDhKHwlWDrbx/6FPH2SuKE+u4msIhPFin4V3FAP+yTem/TKrdnaWy6 +EUrhCWTXVRTijBaCudfjFd/ipHZbA/0dv7UAcoWK6kiVLzyE+jOvtN+ZxTzxq7CW +mlFPgAC966hgJmz9IXqadtMgPAoL3PK9q1DbPM3JhsQcJrNzTJqZrdN1/kPU0HHa ++aof1BVy3wSvp2mXgaRUULStyhUIyBRM6hAYp3/MoWEYn/bwr+zQkIU8Zsk6OsZ6 +q1xE3cowrUWFtCVSU0EgMjA0OCBFbmMgPHJzYTIwNDhlbmNAZXhhbXBsZS5vcmc+ +iQE0BBMBAgAeBQJC69ptAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEMiZ6pNE +GVVZHMkIAJtGHHZ9iM8Yq1rr0zl1L6SvlQP8JCaxHa31wH3PKqGtq2M+cpb2rXf7 +gAY/doHJPXggfVzkyFrysmQ1gPbDGYLyOutw+IkhihEb5bWxQBNj+3zAFs1YX6v2 +HXWbSUSmyY1V9/+NTtKk03olDc/swd3lXzkuUOhcgfpBgIt3Q+MpT6M2+OIF7lVf +Sb1rWdpwTfGhZzW9szQOeoS4gPvxCCRyuabQRJ6DWH61F8fFIDJg1z+A/Obx4fqX +6GOA69RzgZ3oukFBIXxNwV9PZNnAmHtZVYO80g/oVYBbuvOYedffDBeQarhERZ5W +2TnIE+nqY61YOLBqosliygdZTXULzNidA5YEQuvaugEIAOuCJZdkzORA6e1lr81L +nr4JzMsVBFA+X/yIkBbV6qX/A4nVSLAZKNPXz1YIrMTu+1rMIiy10IWbA6zgMTpz +PhJRfgePONgdnCYyK5Ksh5/C5ntzKwwGwxfKlAXIxJurCHXTbEa+YvPdn76vJ3Hs +XOXVEL+fLb4U3l3Ng87YM202Lh1Ha2MeS2zEFZcAoKbFqAAjDLEai64SoOFh0W3C +sD1DL4zmfp+YZrUPHTtZadsi53i4KKW/ws9UrHlolqYNhYze/uRLyfnUx9PN4r/G +hEzauyDMV0smo91uB3aewPft+eCpmeWnu0PFJVK4xyRmhIq2rVCw16a1pBJirvGM ++y0ABikAB/oC3z7lv6sVg+ngjbpWy9lZu2/ECZ9FqViVz7bUkjfvSuowgpncryLW +4EpVV4U6mMSgU6kAi5VGT/BvYGSAtnqDWGiPs7Kk+h4Adz74bEAXzU280pNBtSfX +tGvzlS4a376KzYFSCJDRBdMebEhJMbY0wQmR8lTZu5JSUI4YYEuN0c7ckdsw8w42 +QWTLonG8HC6h8UPKS0EAcaCo7tFubMIesU6cWuTYucsHE+wjbADjuSNX968qczNe +NoL2BUznXOQoPu6HQO4/8cr7ib+VQkB2bHQcMoZazPUStIID1e4CL4XcxfuAmT8o +3XDvMLgVqNp5W2f8Mzmk3/DbtsLXLOv5BADsCzQpseC8ikSYJC72hcon1wlUmGeH +3qgGiiHhYXFa18xgI5juoO8DaWno0rPPlgr36Y8mSB5qjYHMXwjKnKyUmt11H+hU ++6uk4hq3Rjd8l+vfuOSr1xoTrtBUg9Rwfw6JVo0DC+8CWg4oBWsLXVM6KQXPFdJs +8kyFQplR/iP1XQQA/2tbDANjAYGNNDjJO9/0kEnSAUyYMasFJDrA2q17J5CroVQw +QpMmWwdDkRANUVPKnWHS5sS65BRc7UytKe2f3A3ZInGXJIK2Hl+TzapWYcYxql+4 +ol5mEDDMDbhEE8Wmj9KyB6iifdLI0K+yxNb9T4Jpj3J18+St+G8+9AcFcBEEAM1b +M9C+/05cnV8gjcByqH9M9ypo8fzPvMKVXWwCLQXpaL50QIkzLURkiMoEWrCdELaA +sVPotRzePTIQ1ooLeDxd1gRnDqjZiIR0kwmv6vq8tfzY96O2ZbGWFI5eth89aWEJ +WB8AR3zYcXpwJLwPuhXW2/NlZF0bclJ3jNzAfTIeQmeJAR8EGAECAAkFAkLr2roC +GwwACgkQyJnqk0QZVVku1wgAg1bLSjPkhw+ldG5HzumpqR84+JKyozdJaJzefu2+ +1iqYE0B0WLz2PJVIiK41xiEkKhBvTOQYuXmtWqAWXptD91P5SoXoNJWLQO3TNwar +ANhHxkWgw/TOUxQqoctlRUej5NDD+4eW5G9lcS1FEGuKDWtX096u80vO+TbyJjvx +2eVM1k+XdmeYsGOiNgDimCreJGYc14G7eY9jt24gw10n1sMAKI1qm6lcoHqZ9OOy +la+wJdroPYZGO7R8+1O9R22WrK6BYDT5j/1JwMZqbOESjNvDEVT0yOHClCHRN4CC +hbt6LhKhCLUNdz/udIt0JAC6c/HdPLSW3HnmM3+iNj+Kug== +=UKh3 +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (7, 'rsaenc2048-psw', ' +same key with password +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.11 (GNU/Linux) + +lQPEBELr2m0BCADOrnknlnXI0EzRExf/TgoHvK7Xx/E0keWqV3KrOyC3/tY2KOrj +UVxaAX5pkFX9wdQObGPIJm06u6D16CH6CildX/vxG7YgvvKzK8JGAbwrXAfk7OIW +czO2zRaZGDynoK3mAxHRBReyTKtNv8rDQhuZs6AOozJNARdbyUO/yqUnqNNygWuT +4htFDEuLPIJwAbMSD0BvFW6YQaPdxzaAZm3EWVNbwDzjgbBUdBiUUwRdZIFUhsjJ +dirFdy5+uuZru6y6CNC1OERkJ7P8EyoFiZckAIE5gshVZzNuyLOZjc5DhWBvLbX4 +NZElAnfiv+4nA6y8wQLSIbmHA3nqJaBklj85AAYp/gcDCNnoEKwFo86JYCE1J92R +HRQ7DoyAZpW1O0dTXL8Epk0sKsKDrCJOrIkDymsjfyBexADIeqOkioy/50wD2Mku +CVHKWO2duAiJN5t/FoRgpR1/Q11K6QdfqOG0HxwfIXLcPv7eSIso8kWorj+I01BP +Fn/atGEbIjdWaz/q2XHbu0Q3x6Et2gIsbLRVMhiYz1UG9uzGJ0TYCdBa2SFhs184 +52akMpD+XVdM0Sq9/Cx40Seo8hzERB96+GXnQ48q2OhlvcEXiFyD6M6wYCWbEV+6 +XQVMymbl22FPP/bD9ReQX2kjrkQlFAtmhr+0y8reMCbcxwLuQfA3173lSPo7jrbH +oLrGhkRpqd2bYCelqdy/XMmRFso0+7uytHfTFrUNfDWfmHVrygoVrNnarCbxMMI0 +I8Q+tKHMThWgf0rIOSh0+w38kOXFCEqEWF8YkAqCrMZIlJIed78rOCFgG4aHajZR +D8rpXdUOIr/WeUddK25Tu8IuNJb0kFf12IMgNh0nS+mzlqWiofS5kA0TeB8wBV6t +RotaeyDNSsMoowfN8cf1yHMTxli+K1Tasg003WVUoWgUc+EsJ5+KTNwaX5uGv0Cs +j6dg6/FVeVRL9UsyF+2kt7euX3mABuUtcVGx/ZKTq/MNGEh6/r3B5U37qt+FDRbw +ppKPc2AP+yBUWsQskyrxFgv4eSpcLEg+lgdz/zLyG4qW4lrFUoO790Cm/J6C7/WQ +Z+E8kcS8aINJkg1skahH31d59ZkbW9PVeJMFGzNb0Z2LowngNP/BMrJ0LT2CQyLs +UxbT16S/gwAyUpJnbhWYr3nDdlwtC0rVopVTPD7khPRppcsq1f8D70rdIxI4Ouuw +vbjNZ1EWRJ9f2Ywb++k/xgSXwJkGodUlrUr+3i8cv8mPx+fWvif9q7Y5Ex1wCRa8 +8FAj/o+hEbQlUlNBIDIwNDggRW5jIDxyc2EyMDQ4ZW5jQGV4YW1wbGUub3JnPokB +NAQTAQIAHgUCQuvabQIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRDImeqTRBlV +WRzJCACbRhx2fYjPGKta69M5dS+kr5UD/CQmsR2t9cB9zyqhratjPnKW9q13+4AG +P3aByT14IH1c5Mha8rJkNYD2wxmC8jrrcPiJIYoRG+W1sUATY/t8wBbNWF+r9h11 +m0lEpsmNVff/jU7SpNN6JQ3P7MHd5V85LlDoXIH6QYCLd0PjKU+jNvjiBe5VX0m9 +a1nacE3xoWc1vbM0DnqEuID78Qgkcrmm0ESeg1h+tRfHxSAyYNc/gPzm8eH6l+hj +gOvUc4Gd6LpBQSF8TcFfT2TZwJh7WVWDvNIP6FWAW7rzmHnX3wwXkGq4REWeVtk5 +yBPp6mOtWDiwaqLJYsoHWU11C8zYnQPEBELr2roBCADrgiWXZMzkQOntZa/NS56+ +CczLFQRQPl/8iJAW1eql/wOJ1UiwGSjT189WCKzE7vtazCIstdCFmwOs4DE6cz4S +UX4HjzjYHZwmMiuSrIefwuZ7cysMBsMXypQFyMSbqwh102xGvmLz3Z++rydx7Fzl +1RC/ny2+FN5dzYPO2DNtNi4dR2tjHktsxBWXAKCmxagAIwyxGouuEqDhYdFtwrA9 +Qy+M5n6fmGa1Dx07WWnbIud4uCilv8LPVKx5aJamDYWM3v7kS8n51MfTzeK/xoRM +2rsgzFdLJqPdbgd2nsD37fngqZnlp7tDxSVSuMckZoSKtq1QsNemtaQSYq7xjPst +AAYp/gcDCNnoEKwFo86JYAsxoD+wQ0zBi5RBM5EphXTpM1qKxmigsKOvBSaMmr0y +VjHtGY3poyV3t6VboOGCsFcaKm0tIdDL7vrxxwyYESETpF29b7QrYcoaLKMG7fsy +t9SUI3UV2H9uUquHgqHtsqz0jYOgm9tYnpesgQ/kOAWI/tej1ZJXUIWEmZMH/W6d +ATNvZ3ivwApfC0qF5G3oPgBSoIuQ/8I+pN/kmuyNAnJWNgagFhA/2VFBvh5XgztV +NW7G//KpR1scsn140SO/wpGBM3Kr4m8ztl9w9U6a7NlQZ2ub3/pIUTpSzyLBxJZ/ +RfuZI7ROdgDMKmEgCYrN2kfp0LIxnYL6ZJu3FDcS4V098lyf5rHvB3PAEdL6Zyhd +qYp3Sx68r0F4vzk5iAIWf6pG2YdfoP2Z48Pmq9xW8qD9iwFcoz9oAzDEMENn6dfq +6MzfoaXEoYp8cR/o+aeEaGUtYBHiaxQcJYx35B9IhsXXA49yRORK8qdwhSHxB3NQ +H3pUWkfw368f/A207hQVs9yYXlEvMZikxl58gldCd3BAPqHm/XzgknRRNQZBPPKJ +BMZebZ22Dm0qDuIqW4GXLB4sLf0+UXydVINIUOlzg+S4jrwx7eZqb6UkRXTIWVo5 +psTsD14wzWBRdUQHZOZD33+M8ugmewvLY/0Uix+2RorkmB7/jqoZvx/MehDwmCZd +VH8sb2wpZ55sj7gCXxvrfieQD/VeH54OwjjbtK56iYq56RVD0h1az8xDY2GZXeT7 +J0c3BGpuoca5xOFWr1SylAr/miEPxOBfnfk8oZQJvZrjSBGjsTbALep2vDJk8ROD +sdQCJuU1RHDrwKHlbUL0NbGRO2juJGsatdWnuVKsFbaFW2pHHkezKuwOcaAJv7Xt +8LRF17czAJ1uaLKwV8Paqx6UIv+089GbWZi7HIkBHwQYAQIACQUCQuvaugIbDAAK +CRDImeqTRBlVWS7XCACDVstKM+SHD6V0bkfO6ampHzj4krKjN0lonN5+7b7WKpgT +QHRYvPY8lUiIrjXGISQqEG9M5Bi5ea1aoBZem0P3U/lKheg0lYtA7dM3BqsA2EfG +RaDD9M5TFCqhy2VFR6Pk0MP7h5bkb2VxLUUQa4oNa1fT3q7zS875NvImO/HZ5UzW +T5d2Z5iwY6I2AOKYKt4kZhzXgbt5j2O3biDDXSfWwwAojWqbqVygepn047KVr7Al +2ug9hkY7tHz7U71HbZasroFgNPmP/UnAxmps4RKM28MRVPTI4cKUIdE3gIKFu3ou +EqEItQ13P+50i3QkALpz8d08tJbceeYzf6I2P4q6 +=QFm5 +-----END PGP PRIVATE KEY BLOCK----- +'); +-- elg1024 / aes128 +insert into encdata (id, data) values (1, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQEOA9k2z2S7c/RmEAQAgVWW0DeLrZ+1thWJGBPp2WRFL9HeNqqWHbKJCXJbz1Uy +faUY7yxVvG5Eutmo+JMiY3mg23/DgVVXHQZsTWpGvGM6djgUNGKUjZDbW6Nog7Mr +e78IywattCOmgUP9vIwwg3OVjuDCN/nVirGQFnXpJBc8DzWqDMWRWDy1M0ZsK7AD +/2JTosSFxUdpON0DKtIY3GLzmh6Nk3iV0g8VgJKUBT1rhCXuMDj3snm//EMm7hTY +PlnObq4mIhgz8NqprmhooxnU0Kapofb3P3wCHPpU14zxhXY8iKO/3JhBq2uFcx4X +uBMwkW4AdNxY/mzJZELteTL8Tr0s7PISk+owb4URpG3n0jsBc0CVULxrjh5Ejkdw +wCM195J6+KbQxOOFQ0b3uOVvv4dEgd/hRERCOq5EPaFhlHegyYJ7YO842vnSDA== +=PABx +-----END PGP MESSAGE----- +'); +-- elg2048 / blowfish +insert into encdata (id, data) values (2, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQIOAywibh/+XMfUEAf+OINhBngEsw4a/IJIeJvUgv1gTQzBwOdQEuc/runr4Oa8 +Skw/Bj0X/zgABVZLem1a35NHaNwaQaCFwMQ41YyWCu+jTdsiyX/Nw0w8LKKz0rNC +vVpG6YuV7Turtsf8a5lXy1K0SHkLlgxQ6c76GS4gtSl5+bsL2+5R1gSRJ9NXqCQP +OHRipEiYwBPqr5R21ZG0FXXNKGOGkj6jt/M/wh3WVtAhYuBI+HPKRfAEjd/Pu/eD +e1zYtkH1dKKFmp44+nF0tTI274xpuso7ShfKYrOK3saFWrl0DWiWteUinjSA1YBY +m7dG7NZ8PW+g1SZWhEoPjEEEHz3kWMvlKheMRDudnQf/dDyX6kZVIAQF/5B012hq +QyVewgTGysowFIDn01uIewoEA9cASw699jw9IoJp+k5WZXnU+INllBLzQxniQCSu +iEcr0x3fYqNtj9QBfbIqyRcY6HTWcmzyOUeGaSyX76j+tRAvtVtXpraFFFnaHB70 +YpXTjLkp8EBafzMghFaKDeXlr2TG/T7rbwcwWrFIwPqEAUKWN5m97Q3eyo8/ioMd +YoFD64J9ovSsgbuU5IpIGAsjxK+NKzg/2STH7zZFEVCtgcIXsTHTZfiwS98/+1H9 +p1DIDaXIcUFV2ztmcKxh9gt2sXRz1W+x6D8O0k3nanU5yGG4miLKaq18fbcA0BD1 ++NIzAfelq6nvvxYKcGcamBMgLo5JkZOBHvyr6RsAKIT5QYc0QTjysTk9l0Am3gYc +G2pAE+3k +=TBHV +-----END PGP MESSAGE----- +'); +-- elg4096 / aes256 +insert into encdata (id, data) values (3, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQQOA7aFBP0Sjh/5EA/+JCgncc8IZmmRjPStWnGf9tVJhgHTn+smIclibGzs0deS +SPSCitzpblwbUDvu964+/5e5Q1l7rRuNN+AgETlEd4eppv7Swn2ChdgOXxRwukcT +Nh3G+PTFvD4ayi7w1db3qvXIt0MwN4Alt436wJmK1oz2Ka9IcyO+wHWrDy1nSGSx +z5x7YEj+EZPgWc/YAvudqE8Jpzd/OT5zSHN09UFkIAk6NxisKaIstbEGFgpqtoDZ +1SJM84XAdL2IcaJ3YY7k/yzwlawhsakKd4GSd5vWmAwvyzzbSiBMfKsDE16ePLNU +ZBF7CzmlCBPZ7YrFAHLpXBXXkCQvzD2BEYOjse50ZEfJ036T7950Ozcdy1EQbGon +nyQ4Gh0PBpnMcBuiXOceWuYzhlzFOzDtlVKdNTxFRDcbEyW2jo9xQYvCCLnYy8EH +2M7S8jCtVYJBbn63a82ELv+3+kWYcsvBJv2ZVBh4ncrBu9o0P+OYS7ApoOU+j6p2 ++t0RXHksqXS1YiUwYF5KSw09EbYMgNZ9G04Px/PxLU6fSC9iDrGX7Xt3kOUP0mku +C518fPckT0zzRXqfFruJNRzDytW50KxkOQZzU1/Az1YlYN9QzWeU4EtLPb2fftZo +D0qH/ln+f9Op5t6sD2fcxZVECU1b/bFtZsxvwH406YL+UQ7hU/XnZrzVVzODal8P +/j1hg7v7BdJqu1DTp9nFWUuwMFcYAczuXn29IG183NZ7Ts4whDeYEhS8eNoLPX4j +txY12ILD/w/3Q4LoW/hPa6OdfEzsn0U5GLf1WiGmJE1H6ft2U/xUnerc/u0kt+FU +WAisArd4MuKtf7B5Vu/VF3kUdrR0hTniUKUivmC4o1jSId31Dufxj4aadVyldXAr +6TNBcdyragZjxEZ6hsBCYzA0Rd1a8atd6OaQoIEEfAzCu5Ks29pydHErStYGjWJ1 +KA5KPLVvjbHpDmRhlCcm8vgpYQsBYEB5gE9fx5yCTlsVhCB6y23h7hfdMqerDqkO +ZOPsO5h+tiHCdIrQ36sMjuINy1/K2rYcXd+Crh2iHcfidpU9fvDz2ihTRNQlhjuT +0cQZM5JhctEx4VXF4LDctRhit7Hn0iqsk604woQfJVvP8O673xSXT/kBY0A/v9C0 +3C4YoFNeSaKwbfZQ/4u1ZFPJxK2IIJa8UGpyAUewLMlzGVVagljybv/f4Z9ERAhy +huq5sMmw8UPsrJF2TUGHz5WSIwoh0J/qovoQI09I9sdEnFczDvRavMO2Mldy3E5i +exz9oewtel6GOmsZQSYWT/vJzbYMmvHNmNpVwwoKrLV6oI3kyQ80GHBwI1WlwHoK +2iRB0w8q4VVvJeYAz8ZIp380cqC3pfO0uZsrOx4g3k4X0jsB5y7rF5xXcZfnVbvG +DYKcOy60/OHMWVvpw6trAoA+iP+cVWPtrbRvLglTVTfYmi1ToZDDipkALBhndQ== +=L/M/ +-----END PGP MESSAGE----- +'); +-- rsaenc2048 / aes128 +insert into encdata (id, data) values (4, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQEMA/0CBsQJt0h1AQf+JyYnCiortj26P11zk28MKOGfWpWyAhuIgwbJXsdQ+e6r +pEyyqs9GC6gI7SNF6+J8B/gsMwvkAL4FHAQCvA4ZZ6eeXR1Of4YG22JQGmpWVWZg +DTyfhA2vkczuqfAD2tgUpMT6sdyGkQ/fnQ0lknlfHgC5GRx7aavOoAKtMqiZW5PR +yae/qR48mjX7Mb+mLvbagv9mHEgQSmHwFpaq2k456BbcZ23bvCmBnCvqV/90Ggfb +VP6gkSoFVsJ19RHsOhW1dk9ehbl51WB3zUOO5FZWwUTY9DJvKblRK/frF0+CXjE4 +HfcZXHSpSjx4haGGTsMvEJ85qFjZpr0eTGOdY5cFhNJAAVP8MZfji7OhPRAoOOIK +eRGOCkao12pvPyFTFnPd5vqmyBbdNpK4Q0hS82ljugMJvM0p3vJZVzW402Kz6iBL +GQ== +=XHkF +-----END PGP MESSAGE----- +'); +-- rsaenc2048 / aes128 (not from gnupg) +insert into encdata (id, data) values (5, ' +-----BEGIN PGP MESSAGE----- + +wcBMA/0CBsQJt0h1AQgAzxZ8j+OTeZ8IlLxfZ/mVd28/gUsCY+xigWBk/anZlK3T +p2tNU2idHzKdAttH2Hu/PWbZp4kwjl9spezYxMqCeBZqtfGED88Y+rqK0n/ul30A +7jjFHaw0XUOqFNlST1v6H2i7UXndnp+kcLfHPhnO5BIYWxB2CYBehItqtrn75eqr +C7trGzU/cr74efcWagbCDSNjiAV7GlEptlzmgVMmNikyI6w0ojEUx8lCLc/OsFz9 +pJUAX8xuwjxDVv+W7xk6c96grQiQlm+FLDYGiGNXoAzx3Wi/howu3uV40dXfY+jx +3WBrhEew5Pkpt1SsWoFnJWOfJ8GLd0ec8vfRCqAIVdLgAeS7NyawQYtd6wuVrEAj +5SMg4Thb4d+g45RksuGLHUUr4qO9tiXglODa4InhmJfgNuLk+RGz4LXjq8wepEmW +vRbgFOG54+Cf4C/gC+HkreDm5JKSKjvvw4B/jC6CDxq+JoziEe2Z1uEjCuEcr+Es +/eGzeOi36BejXPMHeKxXejj5qBBHKV0pHVhZSgffR0TtlXdB967Yl/5agV0R89hI +7Gw52emfnH4Z0Y4V0au2H0k1dR/2IxXdJEWSTG7Be1JHT59p9ei2gSEOrdBMIOjP +tbYYUlmmbvD49bHfThkDiC+oc9947LgQsk3kOOLbNHcjkbrjH8R5kjII4m/SEZA1 +g09T+338SzevBcVXh/cFrQ6/Et+lyyO2LJRUMs69g/HyzJOVWT2Iu8E0eS9MWevY +Qtrkrhrpkl3Y02qEp/j6M03Yu2t6ZF7dp51aJ5VhO2mmmtHaTnCyCc8Fcf72LmD8 +blH2nKZC9d6fi4YzSYMepZpMOFR65M80MCMiDUGnZBB8sEADu2/iVtqDUeG8mAA= +=PHJ1 +-----END PGP MESSAGE----- +'); +-- successful decrypt +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=1 and encdata.id=1; + pgp_pub_decrypt +----------------- + Secret msg +(1 row) + +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=2 and encdata.id=2; +ERROR: Wrong key or corrupt data +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=3 and encdata.id=3; + pgp_pub_decrypt +----------------- + Secret msg +(1 row) + +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=6 and encdata.id=4; + pgp_pub_decrypt +----------------- + Secret message. +(1 row) + +-- wrong key +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=2 and encdata.id=1; +ERROR: Wrong key +-- sign-only key +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=4 and encdata.id=1; +ERROR: No encryption key found +-- rsa: password-protected secret key, wrong password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), '123') +from keytbl, encdata where keytbl.id=7 and encdata.id=4; +ERROR: Wrong key or corrupt data +-- rsa: password-protected secret key, right password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), 'parool') +from keytbl, encdata where keytbl.id=7 and encdata.id=4; + pgp_pub_decrypt +----------------- + Secret message. +(1 row) + +-- password-protected secret key, no password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=5 and encdata.id=1; +ERROR: Need password for secret key +-- password-protected secret key, wrong password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), 'foo') +from keytbl, encdata where keytbl.id=5 and encdata.id=1; +ERROR: Wrong key or corrupt data +-- password-protected secret key, right password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), 'parool') +from keytbl, encdata where keytbl.id=5 and encdata.id=1; + pgp_pub_decrypt +----------------- + Secret msg +(1 row) + +-- test for a short read from prefix_init +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=6 and encdata.id=5; +ERROR: Wrong key or corrupt data From bb391b7eaf42acf90660d1e5a44a06eb6fcca11c Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 8 Dec 2020 15:22:43 +0900 Subject: [PATCH 05/32] pgcrypto: Detect errors with EVP calls from OpenSSL The following routines are called within pgcrypto when handling digests but there were no checks for failures: - EVP_MD_CTX_size (can fail with -1 as of 3.0.0) - EVP_MD_CTX_block_size (can fail with -1 as of 3.0.0) - EVP_DigestInit_ex - EVP_DigestUpdate - EVP_DigestFinal_ex A set of elog(ERROR) is added by this commit to detect such failures, that should never happen except in the event of a processing failure internal to OpenSSL. Note that it would be possible to use ERR_reason_error_string() to get more context about such errors, but these refer mainly to the internals of OpenSSL, so it is not really obvious how useful that would be. This is left out for simplicity. Per report from Coverity. Thanks to Tom Lane for the discussion. Backpatch-through: 9.5 --- contrib/pgcrypto/openssl.c | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index e983a3803fe0..627ef2552847 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -128,16 +128,24 @@ static unsigned digest_result_size(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; + int result = EVP_MD_CTX_size(digest->ctx); - return EVP_MD_CTX_size(digest->ctx); + if (result < 0) + elog(ERROR, "EVP_MD_CTX_size() failed"); + + return result; } static unsigned digest_block_size(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; + int result = EVP_MD_CTX_block_size(digest->ctx); + + if (result < 0) + elog(ERROR, "EVP_MD_CTX_block_size() failed"); - return EVP_MD_CTX_block_size(digest->ctx); + return result; } static void @@ -145,7 +153,8 @@ digest_reset(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestInit_ex(digest->ctx, digest->algo, NULL); + if (!EVP_DigestInit_ex(digest->ctx, digest->algo, NULL)) + elog(ERROR, "EVP_DigestInit_ex() failed"); } static void @@ -153,7 +162,8 @@ digest_update(PX_MD *h, const uint8 *data, unsigned dlen) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestUpdate(digest->ctx, data, dlen); + if (!EVP_DigestUpdate(digest->ctx, data, dlen)) + elog(ERROR, "EVP_DigestUpdate() failed"); } static void @@ -161,7 +171,8 @@ digest_finish(PX_MD *h, uint8 *dst) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestFinal_ex(digest->ctx, dst, NULL); + if (!EVP_DigestFinal_ex(digest->ctx, dst, NULL)) + elog(ERROR, "EVP_DigestFinal_ex() failed"); } static void From 7871f211e6d247a508d07e5ccbb104033bf09d72 Mon Sep 17 00:00:00 2001 From: Daniel Gustafsson Date: Sat, 25 Sep 2021 11:25:48 +0200 Subject: [PATCH 06/32] pgcrypto: Check for error return of px_cipher_decrypt() This has previously not been a problem (that anyone ever reported), but in future OpenSSL versions (3.0.0), where legacy ciphers are/can be disabled, this is the place where this is reported. So we need to catch the error here, otherwise the higher-level functions would return garbage. The nearby encryption code already handled errors similarly. Author: Peter Eisentraut Reviewed-by: Daniel Gustafsson Discussion: https://www.postgresql.org/message-id/9e9c431c-0adc-7a6d-9b1a-915de1ba3fe7@enterprisedb.com Backpatch-through: 9.6 --- contrib/pgcrypto/px.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/pgcrypto/px.c b/contrib/pgcrypto/px.c index 0f02fb56c4fb..2c6704e25777 100644 --- a/contrib/pgcrypto/px.c +++ b/contrib/pgcrypto/px.c @@ -292,6 +292,7 @@ static int combo_decrypt(PX_Combo *cx, const uint8 *data, unsigned dlen, uint8 *res, unsigned *rlen) { + int err = 0; unsigned bs, i, pad; @@ -317,7 +318,9 @@ combo_decrypt(PX_Combo *cx, const uint8 *data, unsigned dlen, /* decrypt */ *rlen = dlen; - px_cipher_decrypt(c, data, dlen, res); + err = px_cipher_decrypt(c, data, dlen, res); + if (err) + return err; /* unpad */ if (bs > 1 && cx->padding) From a4c60784b4eb5ce879025ef85fd7f244798619ed Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 25 Mar 2025 14:58:44 +0700 Subject: [PATCH 07/32] Revert "Add testcase for FIPS mode operations in pgcrypto" This reverts commit d2559fec2cc2923c9c5a0d18a1a0eff238cc6820. --- contrib/pgcrypto/Makefile | 5 +- contrib/pgcrypto/expected/fips.out | 0 contrib/pgcrypto/expected/fips_1.out | 131 ----------------------- contrib/pgcrypto/expected/fips_2.out | 101 ----------------- contrib/pgcrypto/expected/setup_fips.out | 7 -- contrib/pgcrypto/sql/fips.sql | 40 ------- src/test/regress/init_file | 6 -- 7 files changed, 2 insertions(+), 288 deletions(-) delete mode 100644 contrib/pgcrypto/expected/fips.out delete mode 100644 contrib/pgcrypto/expected/fips_1.out delete mode 100644 contrib/pgcrypto/expected/setup_fips.out delete mode 100644 contrib/pgcrypto/sql/fips.sql diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 36a8403a9fb8..cc802e9ea833 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -30,14 +30,13 @@ DATA = pgcrypto--1.3.sql pgcrypto--1.2--1.3.sql pgcrypto--1.1--1.2.sql \ pgcrypto--1.0--1.1.sql pgcrypto--unpackaged--1.0.sql PGFILEDESC = "pgcrypto - cryptographic functions" -REGRESS_OPTS = --dbname=$(CONTRIB_TESTDB) --init-file=$(top_builddir)/src/test/regress/init_file +REGRESS_OPTS = --dbname=$(CONTRIB_TESTDB) --init-file=$(top_builddir)/src/test/regress/init_file REGRESS = init md5 sha1 hmac-md5 hmac-sha1 blowfish rijndael \ $(CF_TESTS) \ crypt-des crypt-md5 crypt-blowfish crypt-xdes \ pgp-armor pgp-decrypt pgp-encrypt $(CF_PGP_TESTS) \ - pgp-pubkey-decrypt pgp-pubkey-encrypt pgp-info \ - setup_fips fips + pgp-pubkey-decrypt pgp-pubkey-encrypt pgp-info EXTRA_CLEAN = gen-rtab diff --git a/contrib/pgcrypto/expected/fips.out b/contrib/pgcrypto/expected/fips.out deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/contrib/pgcrypto/expected/fips_1.out b/contrib/pgcrypto/expected/fips_1.out deleted file mode 100644 index 04021993b337..000000000000 --- a/contrib/pgcrypto/expected/fips_1.out +++ /dev/null @@ -1,131 +0,0 @@ -ALTER DATABASE contrib_regression SET pgcrypto.fips TO on; -ERROR: FIPS enabled OpenSSL is required for strict FIPS mode (openssl.c:1055) -HINT: Recompile OpenSSL with the FIPS module, or install a FIPS enabled OpenSSL distribution. -\c contrib_regression -SHOW pgcrypto.fips; - pgcrypto.fips ---------------- - off -(1 row) - -CREATE TABLE fipstest (data text, res text, salt text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'data' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO fipstest VALUES ('password', '', ''); -SELECT 'Test digest md5: EXPECTED ERROR FAIL FIPS' as comment; - comment -------------------------------------------- - Test digest md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT digest('santa claus', 'md5'); - digest ------------------------------------- - \x7c4f44266cee2dbfacaa009ef5e167dc -(1 row) - -SELECT 'Test digest sha256: EXPECTED PASS' as comment; - comment ------------------------------------ - Test digest sha256: EXPECTED PASS -(1 row) - -SELECT digest('santa claus', 'sha256'); - digest --------------------------------------------------------------------- - \x675b8f61fc27140b5f06fec4613d8b3d9b913a82074d4c790374558c18d2cb7d -(1 row) - -SELECT 'Test hmac md5: EXPECTED ERROR FAIL FIPS' as comment; - comment ------------------------------------------ - Test hmac md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'md5'); - hmac ------------------------------------- - \xd13b4c8c8a6e9d6236e8cc0b141968c5 -(1 row) - -SELECT 'Test hmac sha256: EXPECTED PASS' as comment; - comment ---------------------------------- - Test hmac sha256: EXPECTED PASS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'sha256'); - hmac --------------------------------------------------------------------- - \xc88fe8ff0541b1bb25abd971fa7642d256a1c0109f7e56875d593a3daaeacf54 -(1 row) - -SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; - comment ------------------------------------- - Test gen_salt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET salt = gen_salt('md5'); -SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------- - Test crypt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET res = crypt(data, salt); -SELECT res = crypt(data, res) AS "worked" FROM fipstest; - worked --------- - t -(1 row) - -SELECT 'Test pgp : EXPECTED PASS' as comment; - comment --------------------------- - Test pgp : EXPECTED PASS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); - pgp_sym_decrypt ------------------ - santa clause -(1 row) - -SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; - comment -------------------------------- - Test pgp : EXPECTED FAIL FIPS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=bf'), 'mypass'); - pgp_sym_decrypt ------------------ - santa clause -(1 row) - -SELECT 'Test raw encrypt : EXPECTED PASS' as comment; - comment ----------------------------------- - Test raw encrypt : EXPECTED PASS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; - raw_aes ------------------------------------- - \xe7ac3ad57bde59c5b78c08805afdb774 -(1 row) - -SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------------- - Test raw encrypt : EXPECTED FAIL FIPS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; - raw_blowfish ------------------------------------- - \x0f4fe496594a2e762fb29a22fc6750e2 -(1 row) - -DROP TABLE fipstest; diff --git a/contrib/pgcrypto/expected/fips_2.out b/contrib/pgcrypto/expected/fips_2.out index 51957b898daa..e69de29bb2d1 100644 --- a/contrib/pgcrypto/expected/fips_2.out +++ b/contrib/pgcrypto/expected/fips_2.out @@ -1,101 +0,0 @@ -ALTER DATABASE contrib_regression SET pgcrypto.fips TO on; -\c contrib_regression -SHOW pgcrypto.fips; - pgcrypto.fips ---------------- - on -(1 row) - -CREATE TABLE fipstest (data text, res text, salt text); -INSERT INTO fipstest VALUES ('password', '', ''); -SELECT 'Test digest md5: EXPECTED ERROR FAIL FIPS' as comment; - comment -------------------------------------------- - Test digest md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT digest('santa claus', 'md5'); -ERROR: Cannot use "md5": -SELECT 'Test digest sha256: EXPECTED PASS' as comment; - comment ------------------------------------ - Test digest sha256: EXPECTED PASS -(1 row) - -SELECT digest('santa claus', 'sha256'); - digest --------------------------------------------------------------------- - \x675b8f61fc27140b5f06fec4613d8b3d9b913a82074d4c790374558c18d2cb7d -(1 row) - -SELECT 'Test hmac md5: EXPECTED ERROR FAIL FIPS' as comment; - comment ------------------------------------------ - Test hmac md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'md5'); -ERROR: Cannot use "md5": -SELECT 'Test hmac sha256: EXPECTED PASS' as comment; - comment ---------------------------------- - Test hmac sha256: EXPECTED PASS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'sha256'); - hmac --------------------------------------------------------------------- - \xc88fe8ff0541b1bb25abd971fa7642d256a1c0109f7e56875d593a3daaeacf54 -(1 row) - -SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; - comment ------------------------------------- - Test gen_salt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET salt = gen_salt('md5'); -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:213) -SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------- - Test crypt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET res = crypt(data, salt); -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:266) -SELECT res = crypt(data, res) AS "worked" FROM fipstest; -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:266) -SELECT 'Test pgp : EXPECTED PASS' as comment; - comment --------------------------- - Test pgp : EXPECTED PASS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) -SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; - comment -------------------------------- - Test pgp : EXPECTED FAIL FIPS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=bf'), 'mypass'); -ERROR: Unsupported cipher algorithm -SELECT 'Test raw encrypt : EXPECTED PASS' as comment; - comment ----------------------------------- - Test raw encrypt : EXPECTED PASS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) -SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------------- - Test raw encrypt : EXPECTED FAIL FIPS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) -DROP TABLE fipstest; diff --git a/contrib/pgcrypto/expected/setup_fips.out b/contrib/pgcrypto/expected/setup_fips.out deleted file mode 100644 index 5ee3b794f464..000000000000 --- a/contrib/pgcrypto/expected/setup_fips.out +++ /dev/null @@ -1,7 +0,0 @@ --- Setup for fips.sql test --- We are setting shared_preload_libraries so that we can set the extension GUC --- 'pgcrypto.fips' immediately after creating the extension on master. --- start_ignore -\! gpconfig -c shared_preload_libraries -v '$libdir/pgcrypto' -\! gpstop -air --- end_ignore diff --git a/contrib/pgcrypto/sql/fips.sql b/contrib/pgcrypto/sql/fips.sql deleted file mode 100644 index d0b0d71d0755..000000000000 --- a/contrib/pgcrypto/sql/fips.sql +++ /dev/null @@ -1,40 +0,0 @@ -ALTER DATABASE contrib_regression SET pgcrypto.fips TO on; -\c contrib_regression - -SHOW pgcrypto.fips; - -CREATE TABLE fipstest (data text, res text, salt text); -INSERT INTO fipstest VALUES ('password', '', ''); - -SELECT 'Test digest md5: EXPECTED ERROR FAIL FIPS' as comment; -SELECT digest('santa claus', 'md5'); - -SELECT 'Test digest sha256: EXPECTED PASS' as comment; -SELECT digest('santa claus', 'sha256'); - -SELECT 'Test hmac md5: EXPECTED ERROR FAIL FIPS' as comment; -SELECT hmac('santa claus', 'aaa', 'md5'); - -SELECT 'Test hmac sha256: EXPECTED PASS' as comment; -SELECT hmac('santa claus', 'aaa', 'sha256'); - -SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; -UPDATE fipstest SET salt = gen_salt('md5'); - -SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; -UPDATE fipstest SET res = crypt(data, salt); -SELECT res = crypt(data, res) AS "worked" FROM fipstest; - -SELECT 'Test pgp : EXPECTED PASS' as comment; -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); - -SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=bf'), 'mypass'); - -SELECT 'Test raw encrypt : EXPECTED PASS' as comment; -SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; - -SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; -SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; - -DROP TABLE fipstest; diff --git a/src/test/regress/init_file b/src/test/regress/init_file index 046050e3fb00..a62de3dd63e2 100644 --- a/src/test/regress/init_file +++ b/src/test/regress/init_file @@ -117,12 +117,6 @@ s/\d+ was concurrently dropped/##### was concurrently dropped/ # Mask out linenumber of the erroring file m/ERROR: ANALYZE cannot merge since not all non-empty leaf partitions have consistent hyperloglog statistics for merge.*/ s/ERROR: ANALYZE cannot merge since not all non-empty leaf partitions have consistent hyperloglog statistics for merge.*/ERROR: ANALYZE cannot merge since not all non-empty leaf partitions have consistent hyperloglog statistics for merge (analyze.c:XXX)/ -m/ERROR: Cannot run ANALYZE MERGE since not all non-empty leaf partitions have available statistics for the merge.*/ -s/ERROR: Cannot run ANALYZE MERGE since not all non-empty leaf partitions have available statistics for the merge.*/ERROR: Cannot run ANALYZE MERGE since not all non-empty leaf partitions have available statistics for the merge (analyze.c:XXX)/ -m/ERROR: invalid partition constraint on "[^"]+".*/ -s/ERROR: invalid partition constraint on "[^"]+".*/ERROR: invalid partition constraint on "[^"]+".*(cdbpartition.c:XXX)/ -m/ERROR: FIPS enabled OpenSSL is required for strict FIPS mode .*/ -s/ERROR: FIPS enabled OpenSSL is required for strict FIPS mode .*/ERROR: FIPS enabled OpenSSL is required for strict FIPS mode (openssl.c:XXX)/ # Mask out OpenSSL behavior change in different version m/ERROR: Cannot use "md5": No such hash algorithm/ From b2763b0ca4731cd776574538f439cb27203afb84 Mon Sep 17 00:00:00 2001 From: wenru yan Date: Thu, 6 Jul 2023 06:22:38 +0000 Subject: [PATCH 08/32] Fix case pg_rewind_fail_missing_xlog When execute inject fault `checkpoint_after_redo_calculated` or 'checkpoint_control_file_updated' in a newly promoted mirror node, connection failed error occurs due to the fatal log 'mirror is being promoted'. It means when connection state is MIRROR_READY but the role change to primary during promoting, the connection will be declined to avoid confusion. Wait for segment promotion finished and accept connection before inject fault. Co-authored-by: Xing Guo higuoxing@gmail.com --- .../expected/pg_rewind_fail_missing_xlog.out | 13 ++++++++++++ .../sql/pg_rewind_fail_missing_xlog.sql | 20 +++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/test/isolation2/expected/pg_rewind_fail_missing_xlog.out b/src/test/isolation2/expected/pg_rewind_fail_missing_xlog.out index a819dceb1393..fdace0ddd306 100644 --- a/src/test/isolation2/expected/pg_rewind_fail_missing_xlog.out +++ b/src/test/isolation2/expected/pg_rewind_fail_missing_xlog.out @@ -2,6 +2,11 @@ -- is removed/recylced in checkpointer, gprecoverseg (based on pg_rewind) would -- would fail. +CREATE OR REPLACE LANGUAGE plpython3u; +CREATE +CREATE OR REPLACE FUNCTION connectSeg(n int, port int, hostname text) RETURNS bool AS $$ import os import subprocess import time for i in range(n): try: subprocess.run(["psql", "-h", str(hostname), "-p", str(port), "postgres", "-Xc", "select 1;"], env={"PGOPTIONS": "-c gp_role=utility", "PATH": os.getenv("PATH")}, check=True) return True except Exception as e: time.sleep(1) raise Exception("wait connection timeout") $$ LANGUAGE plpython3u; +CREATE + CREATE TABLE tst_missing_tbl (a int); CREATE INSERT INTO tst_missing_tbl values(2),(1),(5); @@ -187,6 +192,12 @@ INSERT 3 --------------------------- t (1 row) +-- Wait for the segment promotion finished and accept the connection +3: select connectSeg(600,port,hostname) from gp_segment_configuration where content = 1 and role = 'p'; + connectseg +------------ + t +(1 row) -- Wait for the end of recovery CHECKPOINT completed after the mirror was promoted 3: SELECT gp_inject_fault('checkpoint_after_redo_calculated', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content = 1; gp_inject_fault @@ -247,6 +258,8 @@ server closed the connection unexpectedly -- Cleanup 5: DROP TABLE tst_missing_tbl; DROP +5: DROP FUNCTION connectSeg; +DROP !\retcode gprecoverseg -ar; (exited with code 0) 5: SELECT wait_until_all_segments_synchronized(); diff --git a/src/test/isolation2/sql/pg_rewind_fail_missing_xlog.sql b/src/test/isolation2/sql/pg_rewind_fail_missing_xlog.sql index 9a91a2e7ded8..8419d4429bd9 100644 --- a/src/test/isolation2/sql/pg_rewind_fail_missing_xlog.sql +++ b/src/test/isolation2/sql/pg_rewind_fail_missing_xlog.sql @@ -2,6 +2,23 @@ -- is removed/recylced in checkpointer, gprecoverseg (based on pg_rewind) would -- would fail. +CREATE OR REPLACE LANGUAGE plpython3u; +CREATE OR REPLACE FUNCTION connectSeg(n int, port int, hostname text) RETURNS bool AS $$ +import os +import subprocess +import time +for i in range(n): + try: + subprocess.run(["psql", "-h", str(hostname), "-p", str(port), "postgres", "-Xc", "select 1;"], + env={"PGOPTIONS": "-c gp_role=utility", "PATH": os.getenv("PATH")}, + check=True) + return True + except Exception as e: + time.sleep(1) +raise Exception("wait connection timeout") +$$ +LANGUAGE plpython3u; + CREATE TABLE tst_missing_tbl (a int); INSERT INTO tst_missing_tbl values(2),(1),(5); @@ -85,6 +102,8 @@ INSERT INTO tst_missing_tbl values(2),(1),(5); -- Stop the primary immediately and promote the mirror. 3: SELECT pg_ctl(datadir, 'stop', 'immediate') FROM gp_segment_configuration WHERE role='p' AND content = 1; 3: SELECT gp_request_fts_probe_scan(); +-- Wait for the segment promotion finished and accept the connection +3: select connectSeg(600,port,hostname) from gp_segment_configuration where content = 1 and role = 'p'; -- Wait for the end of recovery CHECKPOINT completed after the mirror was promoted 3: SELECT gp_inject_fault('checkpoint_after_redo_calculated', 'skip', dbid) FROM gp_segment_configuration WHERE role='p' AND content = 1; 3: SELECT gp_wait_until_triggered_fault('checkpoint_after_redo_calculated', 1, dbid) FROM gp_segment_configuration WHERE role = 'p' AND content = 1; @@ -111,6 +130,7 @@ INSERT INTO tst_missing_tbl values(2),(1),(5); -- Cleanup 5: DROP TABLE tst_missing_tbl; +5: DROP FUNCTION connectSeg; !\retcode gprecoverseg -ar; 5: SELECT wait_until_all_segments_synchronized(); !\retcode gpconfig -r wal_keep_segments; From 6116499784ddc0bbbdc371159ee346d3b3fd101a Mon Sep 17 00:00:00 2001 From: Michael Paquier Date: Tue, 22 Mar 2022 13:21:48 +0900 Subject: [PATCH 09/32] Fix failures in SSL tests caused by out-of-tree keys and certificates This issue is environment-sensitive, where the SSL tests could fail in various way by feeding on defaults provided by sslcert, sslkey, sslrootkey, sslrootcert, sslcrl and sslcrldir coming from a local setup, as of ~/.postgresql/ by default. Horiguchi-san has reported two failures, but more advanced testing from me (aka inclusion of garbage SSL configuration in ~/.postgresql/ for all the configuration parameters) has showed dozens of failures that can be triggered in the whole test suite. History has showed that we are not good when it comes to address such issues, fixing them locally like in dd87799, and such problems keep appearing. This commit strengthens the entire test suite to put an end to this set of problems by embedding invalid default values in all the connection strings used in the tests. The invalid values are prefixed in each connection string, relying on the follow-up values passed in the connection string to enforce any invalid value previously set. Note that two tests related to CRLs are required to fail with certain pre-set configurations, but we can rely on enforcing an empty value instead after the invalid set of values. Reported-by: Kyotaro Horiguchi Reviewed-by: Andrew Dunstan, Daniel Gustafsson, Kyotaro Horiguchi Discussion: https://postgr.es/m/20220316.163658.1122740600489097632.horikyota.ntt@gmail.com backpatch-through: 10 --- src/test/ssl/t/001_ssltests.pl | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/src/test/ssl/t/001_ssltests.pl b/src/test/ssl/t/001_ssltests.pl index 67a3a28db6a1..d3896f79464c 100644 --- a/src/test/ssl/t/001_ssltests.pl +++ b/src/test/ssl/t/001_ssltests.pl @@ -97,8 +97,13 @@ switch_server_cert($node, 'server-cn-only'); +# Set of default settings for SSL parameters in connection string. This +# makes the tests protected against any defaults the environment may have +# in ~/.postgresql/. +my $default_ssl_connstr = "sslkey=invalid sslcert=invalid sslrootcert=invalid sslcrl=invalid"; + $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid hostaddr=$SERVERHOSTADDR host=common-name.pg-ssltest.test"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb hostaddr=$SERVERHOSTADDR host=common-name.pg-ssltest.test"; # The server should not accept non-SSL connections. test_connect_fails( @@ -190,7 +195,7 @@ # Check that connecting with verify-full fails, when the hostname doesn't # match the hostname in the server's certificate. $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; test_connect_ok( $common_connstr, @@ -210,7 +215,7 @@ switch_server_cert($node, 'server-multiple-alt-names'); $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR sslmode=verify-full"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR sslmode=verify-full"; test_connect_ok( $common_connstr, @@ -241,7 +246,7 @@ switch_server_cert($node, 'server-single-alt-name'); $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR sslmode=verify-full"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR sslmode=verify-full"; test_connect_ok( $common_connstr, @@ -265,7 +270,7 @@ switch_server_cert($node, 'server-cn-and-alt-names'); $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR sslmode=verify-full"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR sslmode=verify-full"; test_connect_ok( $common_connstr, @@ -285,7 +290,7 @@ # not a very sensible certificate, but libpq should handle it gracefully. switch_server_cert($node, 'server-no-names'); $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; test_connect_ok( $common_connstr, @@ -301,7 +306,7 @@ switch_server_cert($node, 'server-revoked'); $common_connstr = - "user=ssltestuser dbname=trustdb sslcert=invalid hostaddr=$SERVERHOSTADDR host=common-name.pg-ssltest.test"; + "$default_ssl_connstr user=ssltestuser dbname=trustdb hostaddr=$SERVERHOSTADDR host=common-name.pg-ssltest.test"; # Without the CRL, succeeds. With it, fails. test_connect_ok( @@ -335,7 +340,7 @@ note "running server tests"; $common_connstr = - "sslrootcert=ssl/root+server_ca.crt sslmode=require dbname=certdb hostaddr=$SERVERHOSTADDR"; + "$default_ssl_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require dbname=certdb hostaddr=$SERVERHOSTADDR"; # no client cert test_connect_fails( @@ -400,7 +405,7 @@ # works, iff username matches Common Name # fails, iff username doesn't match Common Name. $common_connstr = - "sslrootcert=ssl/root+server_ca.crt sslmode=require dbname=verifydb hostaddr=$SERVERHOSTADDR"; + "$default_ssl_connstr sslrootcert=ssl/root+server_ca.crt sslmode=require dbname=verifydb hostaddr=$SERVERHOSTADDR"; test_connect_ok( $common_connstr, @@ -426,7 +431,7 @@ # intermediate client_ca.crt is provided by client, and isn't in server's ssl_ca_file switch_server_cert($node, 'server-cn-only', 'root_ca'); $common_connstr = - "user=ssltestuser dbname=certdb sslkey=ssl/client_tmp.key sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; + "$default_ssl_connstr user=ssltestuser dbname=certdb sslkey=ssl/client_tmp.key sslrootcert=ssl/root+server_ca.crt hostaddr=$SERVERHOSTADDR"; test_connect_ok( $common_connstr, From 2747b4f3b36c5666859197784912ba4889c43455 Mon Sep 17 00:00:00 2001 From: Peter Eisentraut Date: Fri, 5 Jun 2020 11:18:11 +0200 Subject: [PATCH 10/32] Backpatch OpenSSL 3.0.0 compatibility in tests backport of commit f0d2c65f17 to releases 11 and 12 This means the SSL tests will fail on machines with extremely old versions of OpenSSL, but we don't know of anything trying to run such tests. The ability to build is not affected. Discussion: https://postgr.es/m/73e646d3-8653-1a1c-0a39-739872b591b0@dunslane.net --- src/test/ssl/Makefile | 2 +- src/test/ssl/ssl/server-password.key | 52 ++++++++++++++-------------- 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/src/test/ssl/Makefile b/src/test/ssl/Makefile index d1129386a98f..87fc949913de 100644 --- a/src/test/ssl/Makefile +++ b/src/test/ssl/Makefile @@ -74,7 +74,7 @@ ssl/server-ss.crt: ssl/server-cn-only.key ssl/server-cn-only.crt server-cn-only. # Password-protected version of server-cn-only.key ssl/server-password.key: ssl/server-cn-only.key - openssl rsa -des -in $< -out $@ -passout 'pass:secret1' + openssl rsa -aes256 -in $< -out $@ -passout 'pass:secret1' # Client certificate, signed by the client CA: ssl/client.crt: ssl/client.key ssl/client_ca.crt diff --git a/src/test/ssl/ssl/server-password.key b/src/test/ssl/ssl/server-password.key index 337054fe4f0c..a8e383a9498b 100644 --- a/src/test/ssl/ssl/server-password.key +++ b/src/test/ssl/ssl/server-password.key @@ -1,30 +1,30 @@ -----BEGIN RSA PRIVATE KEY----- Proc-Type: 4,ENCRYPTED -DEK-Info: DES-CBC,6FCB918813C446D2 +DEK-Info: AES-256-CBC,B335CBE53A05F4FC5805FC038BA80BA0 -rNHoFsQpQh1UGbh2QSgtWj/pkoe29Q+uZgEwbCTIfE/ONMoiZLC6f2g+PDPg2qYn -BC4VY2NbxkOLw7kK3k6cHwRD5YWfzm+HxmXQYsfPCeGV+C/Pse8srswh7GERSQ77 -zEn7BzkXnevL8ytYAoqezYjXaDSh+76KSN64R21zaGvgToQRGICCevbnjnNRI+HJ -sPGGFTRIWLNNnxULHYpFbMXoJSnOrCbpzEqshUFkdGTpLvLW9se3/LVMWy/Wssje -GwQbh9mDavUlFUl+vTn+ER4C0pzV10DjtQzkDhx88OIhdTsPQ8ClIvVFpBI6fjME -je1n4YkLWGfJrXpifg8hAedmQ9tiw/E+h8IjZJfUgL+C2QybCQ8Yx1/v8HUYzwkA -O7WMM90qI2sxzKWJZVvRVIY7zaTzFcvhpZrceX+d7iLV3h9UF4ru0CcUqK13t3ax -4oOvQnj8DP0fThQOloQX4zY1oUqgLNFdoqgwMIssHzCdsyZ5VdFz8iWA1n+syzKC -DGPJHo4DQ/YNpy0gGmBUcHaiyeKsaDYBQBwE7xm3UnrtYCr1WKkcYfncuNXCZdDJ -X8qiB1w6diDKor9NC15uBBVjv3ERCx1rgG98+KvenQob9dtd6iguLjWtUsQoGI8D -mgcwIuw4NRMIxZa5+1ITOkjnUfvXGTVHwvjHzz2BK6gFQu2NgwgZyrgqSMOZAn3T -JcDDEhE8lgAsEPHVsPiLuc7DbH/ehkv8VocqJE7K9vmdR2UVG2wOO7p/1Ewm0RfJ -9wsry4Ae0uqH+ay/TfMUBOC6tEqcPzicCMc9mYEOdnAjPwoDFFXLNdbjV8WG1KTr -zzn9cucFvbyDOKecWbrduTbEFU5LbTddlOwWBaYVlS8xDX8wYTTyK0HnpLtP5KRJ -69D+CYOnYAQyiPqRnZglIo4hSbNsiWThhWUlbGBWwynhR6fX4MA8PcoFnR88q3g3 -Wjmq0A0jIETwBxYrLFwvgi/PtQJ2p+yjhHCsl3idBgnGYtUiRxpCkuoCIBV1B23j -CJB3NVoOOVEoGi9WYonLOt6vciYQhkjTkM4u3JP6rB/9kDIRpW4HeXYzrhmo0JA2 -PtRbvEwJyMHNoyNc/Pf1ydb+lqv1TBUbPQLsNbzHLKklfAMhQGd3OJWn8NcGB4i7 -uXAHSA/zY5AuE4Vf6mdjCznKbqCGurT+CUctFBajLcV2gwaFT/DikzlzR+TdFJ8E -Qcg+5XUzg7E7tTrLGjM/DFES7ECkpr1cYGheCgoZLj0lT3bLURUUHG4e6BnyGS6M -AdeKh4mZL3HAcbI8rSyoTx0cszQu3BR4QeAMJcnsq69uuLZ3A/r5zYDTIerz53ZO -JYGhWWEsRszToO5eGGKpONIPTIYRLtj5NBfoQ7PZGRfRBlX44Vv3JytsBjrpABJI -iqge5jifhTKcWVB0/OAu3A3uKXOU7AozXa79sdTVRs2KoVAgRTDK7zy3FsYRsT03 -sBdZgR1pr90PhDjlJ6N3Ebk9MXbB9bcwx0C6C/aZPFTsXT6sEoPqlrd6gCLc/YU/ -uTaXmARj45I89uRcZtroq5dZy5HE5nBpds7Vydiz1MwBnIMyBRjjAg== +1U4+GwI8FGpcrk+9uzMlQU5UZ9xOJMOZb9xA3IYMw+2BLF7zVbAkYyyiqF2pKUmi +doOYFOGIXNV1VhVwlw674SMN+PIg72b2F7DDrqEYlicLCU4o7eeGhoiIKzTksRTU +YV3nYCCDZCEw7V+pFeGCUAc9fc+Y0BGMYIshLVdlTYgVjZScL5kHuD9t8xa6AaTS +mQp3jInRnHjEJbRSZnFQ9CR1LUtmGE02TOcWzoGshFdwCdtO/lJzBmmMxoL/qV1R +Cqc0PKSANsbgvTJMriZXYSFjpMYXmxBQXDYNuFfwq67bssAVIpTSvWu9SfcY/JwV +OqERcb1zPgDmprDvd/L7Vh/cdEWWWewOVoUo89cT7CrLvMINHqE6smM2x1xv91BB +AOpyoGJliPGAcLDVJINm9zC1ErEjSEcR/VumZKsgSTsBYgyYezTPQYAfe+h820rs +eC4GMu+zr31U2TVLYcb4j2t19fTgaQBj/LH3OBse9+0quoJhzmDjKelS3O3BaF05 +DM20tJRHANM+1WQ9+aFinXa1ozcGsrLSUa99oFqL4vKgL7jd0+wmCzwxaSp3rHB3 +AFHCdUOayDAdPhnGwathhAZ0AjyEJyWnA47pEpWDr7SytpbiMwOoPcW8/oKid10e +qBK7uGK1Zc7rtckjK3CrM1VFDbxzwGbF2aKHtFFyrJtUvJwfP0Y1V2DncOsiy5Nx +gJ3vxfi11gxnhd9VmcoY3JVvTHOsw48xYNFrZXve/X3o9eUDqb9VRs/vV3t5w+xR +RaUPdz9cdlp2AA4xW/IvIQ7XwuBWPaPVr/g9pUvI9iJ9Z4RdruvjqDAD+ICVx9MM +8SuN7X3gmg4mF5FEL0ct5ZdP16U8/EYvl7Np7vN3kYqbqucwCJH15R8LckAfbzIH +yYTXC1iik4GfyN9tTpQtZsZCvV2Uo+Fo3mxP/EzB6tNbfOi3LG/coverSwgZLQsA +Q6+Kta4PT671xXdaGLT9tEMIai9SiW5acqcdhjYvcaP69J8ZtKpNpP6HTL7IZD8p +SbMxE9jw+bYXILR3Ie0x98z4Z04Q28/bPbvPTbXK8nv6/YpjKgq4hrRG58psHdbX +ggS3RNzcJJMDArBka+zvbWL4jfWZhllMyGqc7q/FuoEqC5JlMTUBpru3NTNp6ZgQ +QXRV1Pc02ff8Dp1H8FP7B7bG3E2D9eTUqR60WvmGnuAqvXgA0+4rEaUKfxELH5qc +dZgu/yiuMttCha835wMLnOxsOJmHILwrc6/uQWydx3vNEWFx0tbV3FzVBIvqdpME +LA4iAAz5xqvLgA5ii23Hn18ycZGU7gTERK8RdiALRzPtBW6hPreQjiMTJnBaMhXA +Xq9opGsNmH/rZgXuk2VZ79bbl9pKN+z9ssRGzbHCVlEckfaxlrYfANwzk8PbOrZJ +6UW3Gf2PwRRNtiVEabf0upVng7V70KSRzjfC7KBHYwbRIL4nObgTG+vc1SjgNgrx +Ue/e8h9qiDBmgdH0Uvqfqb19HF+QzmUNoP9TVQFj+4+DuW5zN0D8weF4TuBgyHr6 +Y+Rbmq0WJlIlc8KMwX87nACesmFNSJkI0ftSLDHrLuvXRtB8f7s2cw3hd81i+scE -----END RSA PRIVATE KEY----- From 6a4e3fed8877958e041f2dfba8d7198b2fb658dd Mon Sep 17 00:00:00 2001 From: HouLei Date: Mon, 30 Jan 2023 17:09:38 +0800 Subject: [PATCH 11/32] Add ZSTD compression support for gpfdist writable external table (#14250) In the previous https://github.com/greenplum-db/gpdb/pull/14144, we finished the support of compression transmission for readable gpfdist external table. In this PR, I developed the code for zstd compression transmission for the writable gpfdist external table. The pr involves three aspects of changes. The first part is adding decompression code for gpfdist. The second part is adding compression code for gpdb. Other modifications mainly involve the regression test for writable gpfdist external table. Finally, to use the zstd compression to transfer data for a writable external table, we need to use the flag --compress to turn on the compression transmission for gpfdist. --- src/backend/access/external/url_curl.c | 72 +++++- src/bin/gpfdist/gpfdist.c | 236 +++++++++++++++--- .../regress/input/gpfdist2_compress.source | 128 +++++++++- .../regress/output/gpfdist2_compress.source | 135 ++++++++++ 4 files changed, 525 insertions(+), 46 deletions(-) diff --git a/src/backend/access/external/url_curl.c b/src/backend/access/external/url_curl.c index f247a2186f55..0e530170bdf5 100644 --- a/src/backend/access/external/url_curl.c +++ b/src/backend/access/external/url_curl.c @@ -46,7 +46,8 @@ typedef struct curlhandle_t { CURL *handle; /* The curl handle */ #ifdef USE_ZSTD - ZSTD_DCtx *zstd_dctx; /* The zstd context */ + ZSTD_DCtx *zstd_dctx; /* The zstd decompression context */ + ZSTD_CCtx *zstd_cctx; /* The zstd compression context */ #endif struct curl_slist *x_httpheader; /* list of headers */ bool in_multi_handle; /* T, if the handle is in global @@ -85,7 +86,8 @@ typedef struct struct { - char *ptr; /* palloc-ed buffer */ + char *ptr; /* palloc-ed buffer */ + char *cptr; int max; int bot, top; @@ -118,7 +120,6 @@ typedef struct #define HOST_NAME_SIZE 100 #define FDIST_TIMEOUT 408 #define MAX_TRY_WAIT_TIME 64 - /* * SSL support GUCs - should be added soon. Until then we will use stubs * @@ -188,6 +189,7 @@ static curlhandle_t *open_curl_handles; static bool url_curl_resowner_callback_registered; + static curlhandle_t * create_curlhandle(void) { @@ -200,6 +202,7 @@ create_curlhandle(void) #ifdef USE_ZSTD h->zstd_dctx = NULL; + h->zstd_cctx = NULL; #endif h->owner = CurrentResourceOwner; @@ -251,6 +254,11 @@ destroy_curlhandle(curlhandle_t *h) ZSTD_freeDCtx(h->zstd_dctx); h->zstd_dctx = NULL; } + if (h->zstd_cctx) + { + ZSTD_freeCCtx(h->zstd_cctx); + h->zstd_cctx = NULL; + } #endif pfree(h); } @@ -400,7 +408,15 @@ header_callback(void *ptr_, size_t size, size_t nmemb, void *userp) buf[i] = 0; #ifdef USE_ZSTD url->zstd = strtol(buf, 0, 0); - if (!url->for_write && url->zstd) + + if (url->for_write && url->zstd) + { + url->curl->zstd_cctx = ZSTD_createCCtx(); + // allocate out.cptr whose size equals to out.ptr + url->out.cptr = (char *) palloc(writable_external_table_bufsize * 1024); + url->lastsize = 0; + } + else if (url->zstd) { url->curl->zstd_dctx = ZSTD_createDCtx(); url->lastsize = ZSTD_initDStream(url->curl->zstd_dctx); @@ -1258,6 +1274,9 @@ url_curl_fopen(char *url, bool forwrite, extvar_t *ev, CopyState pstate) set_httpheader(file, "X-GP-SEGMENT-COUNT", ev->GP_SEGMENT_COUNT); set_httpheader(file, "X-GP-LINE-DELIM-STR", ev->GP_LINE_DELIM_STR); set_httpheader(file, "X-GP-LINE-DELIM-LENGTH", ev->GP_LINE_DELIM_LENGTH); +#ifdef USE_ZSTD + set_httpheader(file, "X-GP-ZSTD", "1"); +#endif if (forwrite) { @@ -1278,9 +1297,6 @@ url_curl_fopen(char *url, bool forwrite, extvar_t *ev, CopyState pstate) { /* read specific - (TODO: unclear why some of these are needed) */ set_httpheader(file, "X-GP-PROTO", "1"); -#ifdef USE_ZSTD - set_httpheader(file, "X-GP-ZSTD", "1"); -#endif set_httpheader(file, "X-GP-MASTER_HOST", ev->GP_MASTER_HOST); set_httpheader(file, "X-GP-MASTER_PORT", ev->GP_MASTER_PORT); set_httpheader(file, "X-GP-CSVOPT", ev->GP_CSVOPT); @@ -1485,6 +1501,13 @@ url_curl_fclose(URL_FILE *fileg, bool failOnError, const char *relname) file->out.ptr = NULL; } + if (file->out.cptr) + { + Assert(file->for_write); + pfree(file->out.cptr); + file->out.cptr = NULL; + } + file->gp_proto = 0; file->error = file->eof = 0; memset(&file->in, 0, sizeof(file->in)); @@ -1563,6 +1586,12 @@ decompress_zstd_data(ZSTD_DCtx* ctx, ZSTD_inBuffer* bin, ZSTD_outBuffer* bout) } return ret; } + +static int +compress_zstd_data(URL_CURL_FILE *file) +{ + return ZSTD_compressCCtx(file->curl->zstd_cctx, file->out.cptr, file->out.top, file->out.ptr, file->out.top, file->zstd); +} #endif /* @@ -1819,6 +1848,7 @@ gp_proto1_read(char *buf, int bufsz, URL_CURL_FILE *file, CopyState pstate, char return n; } + /* * gp_proto0_write * @@ -1827,9 +1857,20 @@ gp_proto1_read(char *buf, int bufsz, URL_CURL_FILE *file, CopyState pstate, char */ static void gp_proto0_write(URL_CURL_FILE *file, CopyState pstate) -{ - char* buf = file->out.ptr; - int nbytes = file->out.top; +{ + char* buf; + int nbytes; +#ifdef USE_ZSTD + if(file->zstd){ + nbytes = compress_zstd_data(file); + buf = file->out.cptr; + } + else +#endif + { + buf = file->out.ptr; + nbytes = file->out.top; + } if (nbytes == 0) return; @@ -1926,11 +1967,18 @@ curl_fwrite(char *buf, int nbytes, URL_CURL_FILE *file, CopyState pstate) char* newbuf; newbuf = repalloc(file->out.ptr, n); - if (!newbuf) elog(ERROR, "out of memory (curl_fwrite)"); - file->out.ptr = newbuf; + + if (file->zstd) + { + newbuf = repalloc(file->out.cptr, n); + if (!newbuf) + elog(ERROR, "out of compress memory (curl_fwrite)"); + file->out.cptr = newbuf; + } + file->out.max = n; Assert(nbytes < file->out.max); diff --git a/src/bin/gpfdist/gpfdist.c b/src/bin/gpfdist/gpfdist.c index 7632a2314446..c68c8c68ea92 100644 --- a/src/bin/gpfdist/gpfdist.c +++ b/src/bin/gpfdist/gpfdist.c @@ -88,6 +88,14 @@ struct block_t char* cdata; }; +typedef struct zstd_buffer zstd_buffer; +struct zstd_buffer +{ + char *buf; + int size; + int pos; +}; + /* Get session id for this request */ #define GET_SID(r) ((r->sid)) @@ -293,17 +301,23 @@ struct request_t char* dbuf; /* buffer for raw data from a POST request */ int dbuftop; /* # bytes used in dbuf */ int dbufmax; /* size of dbuf[] */ + + char* wbuf; /* data buf for decompressed data for writing into file, + its capacity equals to MAX_FRAME_SIZE. */ + int wbuftop; /* last index for decompressed data */ + int woffset; /* mark whether there is left data in compress ctx */ } in; block_t outblock; /* next block to send out */ char* line_delim_str; int line_delim_length; #ifdef USE_ZSTD - ZSTD_CCtx* zstd_cctx; /* zstd context */ + ZSTD_CCtx* zstd_cctx; /* zstd compression context */ + ZSTD_DCtx* zstd_dctx; /* zstd decompression context */ #endif - int zstd; /* request use zstd compress */ - int zstd_err_len; /* space allocate for zstd_error string */ - char* zstd_error; /* string contains zstd error*/ + int zstd; /* request use zstd compress */ + int zstd_err_len; /* space allocate for zstd_error string */ + char* zstd_error; /* string contains zstd error*/ #ifdef USE_SSL /* SSL related */ BIO *io; /* for the i.o. */ @@ -365,7 +379,10 @@ static int request_validate(request_t *r); static int request_set_path(request_t *r, const char* d, char* p, char* pp, char* path); static int request_path_validate(request_t *r, const char* path); #ifdef USE_ZSTD -static int compress_zstd(const request_t *r, block_t *blk, int buflen); +static int compress_zstd(const request_t *r, block_t* block, int buflen); +static int decompress_data(request_t *r, zstd_buffer *in, zstd_buffer *out); +static int decompress_zstd(request_t* r, ZSTD_inBuffer* bin, ZSTD_outBuffer* bout); +static int decompress_write_loop(request_t *r); #endif static int request_parse_gp_headers(request_t *r, int opt_g); static void free_session_cb(int fd, short event, void* arg); @@ -3043,6 +3060,47 @@ static void handle_get_request(request_t *r) } } +static +int check_output_to_file(request_t *r, int wrote) +{ + session_t *session = r->session; + char *buf; + int *buftop; + if (r->zstd) + { + buf = r->in.wbuf; + buftop = &r->in.wbuftop; + } + else + { + buf = r->in.dbuf; + buftop = &r->in.dbuftop; + } + + if (wrote == -1) + { + /* write error */ + gwarning(r, "handle_post_request, write error: %s", fstream_get_error(session->fstream)); + http_error(r, FDIST_INTERNAL_ERROR, fstream_get_error(session->fstream)); + request_end(r, 1, 0); + return -1; + } + else if(wrote == *buftop) + { + /* wrote the whole buffer. clean it for next round */ + *buftop = 0; + } + else + { + /* wrote up to last line, some data left over in buffer. move to front */ + int bytes_left_over = *buftop - wrote; + + memmove(buf, buf + wrote, bytes_left_over); + *buftop = bytes_left_over; + } + return 0; +} + static void handle_post_request(request_t *r, int header_end) { int h_count = r->in.req->hc; @@ -3139,7 +3197,10 @@ static void handle_post_request(request_t *r, int header_end) /* create a buffer to hold the incoming raw data */ r->in.dbufmax = opt.m; /* size of max line size */ r->in.dbuftop = 0; + r->in.wbuftop = 0; r->in.dbuf = palloc_safe(r, r->pool, r->in.dbufmax, "out of memory when allocating r->in.dbuf: %d bytes", r->in.dbufmax); + if(r->zstd) + r->in.wbuf = palloc_safe(r, r->pool, MAX_FRAME_SIZE, "out of memory when allocating r->in.wbuf: %d bytes", MAX_FRAME_SIZE); /* if some data come along with the request, copy it first */ data_start = strstr(r->in.hbuf, "\r\n\r\n"); @@ -3153,21 +3214,35 @@ static void handle_post_request(request_t *r, int header_end) { /* we have data after the request headers. consume it */ /* should make sure r->in.dbuftop + data_bytes_in_req < r->in.dbufmax */ + memcpy(r->in.dbuf, data_start, data_bytes_in_req); r->in.dbuftop += data_bytes_in_req; + + r->in.davailable -= data_bytes_in_req; /* only write it out if no more data is expected */ if(r->in.davailable == 0) { - wrote = fstream_write(session->fstream, r->in.dbuf, data_bytes_in_req, 1, r->line_delim_str, r->line_delim_length); - delay_watchdog_timer(); - if(wrote == -1) +#ifdef USE_ZSTD + if(r->zstd) { - /* write error */ - http_error(r, FDIST_INTERNAL_ERROR, fstream_get_error(session->fstream)); - request_end(r, 1, 0); - return; + wrote = decompress_write_loop(r); + if (wrote == -1) + return; + } + else +#endif + { + wrote = fstream_write(session->fstream, r->in.dbuf, data_bytes_in_req, 1, r->line_delim_str, r->line_delim_length); + delay_watchdog_timer(); + if (wrote == -1) + { + /* write error */ + http_error(r, FDIST_INTERNAL_ERROR, fstream_get_error(session->fstream)); + request_end(r, 1, 0); + return; + } } } } @@ -3179,7 +3254,7 @@ static void handle_post_request(request_t *r, int header_end) while(r->in.davailable > 0) { size_t want; - ssize_t n; + ssize_t n = 0; size_t buf_space_left = r->in.dbufmax - r->in.dbuftop; if (r->in.davailable > buf_space_left) @@ -3223,36 +3298,33 @@ static void handle_post_request(request_t *r, int header_end) r->in.davailable -= n; r->in.dbuftop += n; + /* success is a flag to check whether data is written into file successfully. + * There is no need to do anything when success is less than 0, since all + * error handling has been done in 'check_output_to_file' function. + */ + int success = 0; + /* if filled our buffer or no more data expected, write it */ if (r->in.dbufmax == r->in.dbuftop || r->in.davailable == 0) { +#ifdef USE_ZSTD /* only write up to end of last row */ - wrote = fstream_write(session->fstream, r->in.dbuf, r->in.dbuftop, 1, r->line_delim_str, r->line_delim_length); - gdebug(r, "wrote %d bytes to file", wrote); - delay_watchdog_timer(); - - if (wrote == -1) + if(r->zstd) { - /* write error */ - gwarning(r, "handle_post_request, write error: %s", fstream_get_error(session->fstream)); - http_error(r, FDIST_INTERNAL_ERROR, fstream_get_error(session->fstream)); - request_end(r, 1, 0); - return; - } - else if(wrote == r->in.dbuftop) - { - /* wrote the whole buffer. clean it for next round */ - r->in.dbuftop = 0; + success = decompress_write_loop(r); } else +#endif { - /* wrote up to last line, some data left over in buffer. move to front */ - int bytes_left_over = r->in.dbuftop - wrote; + wrote = fstream_write(session->fstream, r->in.dbuf, r->in.dbuftop, 1, r->line_delim_str, r->line_delim_length); + gdebug(r, "wrote %d bytes to file", wrote); + delay_watchdog_timer(); - memmove(r->in.dbuf, r->in.dbuf + wrote, bytes_left_over); - r->in.dbuftop = bytes_left_over; + success = check_output_to_file(r, wrote); } } + if (success < 0) + return; } } @@ -3489,6 +3561,9 @@ static int request_parse_gp_headers(request_t *r, int opt_g) #ifdef USE_ZSTD if (r->zstd) { + if (!r->is_get) + r->zstd_dctx = ZSTD_createDCtx(); + OUT_BUFFER_SIZE = ZSTD_CStreamOutSize(); r->zstd_err_len = 1024; r->outblock.cdata = palloc_safe(r, r->pool, opt.m, "out of memory when allocating buffer for compressed data: %d bytes", opt.m); @@ -4488,6 +4563,10 @@ static void request_cleanup(request_t *r) { ZSTD_freeCCtx(r->zstd_cctx); } + if ( r->zstd && !r->is_get ) + { + ZSTD_freeDCtx(r->zstd_dctx); + } #endif } @@ -4597,6 +4676,7 @@ static void delay_watchdog_timer() shutdown_time = apr_time_now() + gcb.wdtimer * APR_USEC_PER_SEC; } } + #else static void delay_watchdog_timer() { @@ -4604,6 +4684,96 @@ static void delay_watchdog_timer() #endif #ifdef USE_ZSTD + +/* decompress the data and write data to the file. + * Finally, the function will check the write result, + * and change the related value about data buffer. + */ +static +int decompress_write_loop(request_t *r) +{ + session_t *session = r->session; + int wrote_total = 0; + do + { + int offset = 0; + if (r->in.woffset) + offset = r->in.woffset; + + zstd_buffer in = {r->in.dbuf, r->in.dbuftop, offset}; + zstd_buffer out = {r->in.wbuf + r->in.wbuftop, MAX_FRAME_SIZE - r->in.wbuftop, 0}; + + int res = decompress_data(r, &in, &out); + + if (res < 0) + { + http_error(r, FDIST_INTERNAL_ERROR, r->zstd_error); + request_end(r, 1, 0); + return res; + } + + int wrote = fstream_write(session->fstream, r->in.wbuf, r->in.wbuftop, 0, r->line_delim_str, r->line_delim_length); + wrote_total += wrote; + gdebug(r, "wrote %d bytes to file", wrote); + delay_watchdog_timer(); + + res = check_output_to_file(r, wrote); + if (res < 0) + { + return -1; + } + + } while(r->in.woffset); + return wrote_total; +} + +static int decompress_zstd(request_t* r, ZSTD_inBuffer* bin, ZSTD_outBuffer* bout) +{ + int ret; + /* The return code is zero if the frame is complete, but there may + * be multiple frames concatenated together. Zstd will automatically + * reset the context when a frame is complete. Still, calling + * ZSTD_DCtx_reset() can be useful to reset the context to a clean + * state, for instance if the last decompression call returned an + * error. + */ + + ret = ZSTD_decompressStream(r->zstd_dctx, bout, bin); + size_t const err = ret; + if(ZSTD_isError(err)){ + snprintf(r->zstd_error, r->zstd_err_len, "zstd decompression error, error is %s", ZSTD_getErrorName(err)); + gwarning(NULL, "%s", r->zstd_error); + return -1; + } + return bout->pos; +} + +static int decompress_data(request_t* r, zstd_buffer *in, zstd_buffer *out){ + ZSTD_inBuffer inbuf = {in->buf , in->size, in->pos}; + ZSTD_outBuffer obuf = {out->buf, out->size, out->pos}; + + if(!r->zstd_dctx) { + gwarning(stderr, "%s", "Out of memory when ZSTD_createDCtx"); + return -1; + } + + int outSize = decompress_zstd(r, &inbuf, &obuf); + if(outSize < 0){ + return outSize; + } + + r->in.wbuftop += outSize; + if (inbuf.pos == inbuf.size) + { + r->in.woffset = 0; + } + else + { + r->in.woffset = inbuf.pos; + } + gdebug(NULL, "decompress_zstd finished, input size = %d, output size = %d.", r->in.wbuftop, r->in.dbuftop); + return outSize; +} /* * compress_zstd * It is for compress data in buffer. Return is the length of data after compression. @@ -4660,6 +4830,8 @@ static int compress_zstd(const request_t *r, block_t *blk, int buflen) } offset += output.pos; + gdebug(NULL, "compress_zstd finished, input size = %d, output size = %d.", buflen, offset); + return offset; } #endif diff --git a/src/bin/gpfdist/regress/input/gpfdist2_compress.source b/src/bin/gpfdist/regress/input/gpfdist2_compress.source index 8c249ca6d74f..ba9379fc9e57 100644 --- a/src/bin/gpfdist/regress/input/gpfdist2_compress.source +++ b/src/bin/gpfdist/regress/input/gpfdist2_compress.source @@ -55,7 +55,69 @@ FORMAT 'text' DELIMITER AS '|' ) ; +CREATE TABLE lineitem (like ext_lineitem); SELECT count(*) FROM ext_lineitem; +INSERT INTO lineitem SELECT * FROM ext_lineitem; +DROP EXTERNAL TABLE ext_lineitem; +--test 1.1 test writable table using compression +CREATE WRITABLE EXTERNAL TABLE ext_lineitem_w ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE EXTERNAL TABLE ext_lineitem ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +INSERT INTO ext_lineitem_w SELECT * FROM lineitem; +DROP TABLE lineitem; +SELECT count(*) FROM ext_lineitem; +DROP EXTERNAL TABLE ext_lineitem_w; DROP EXTERNAL TABLE ext_lineitem; -- test 2 use a bigger file. @@ -87,7 +149,70 @@ FORMAT 'text' DELIMITER AS '|' ) ; +CREATE TABLE lineitem (like ext_lineitem); SELECT count(*) FROM ext_lineitem; +INSERT INTO lineitem SELECT * FROM ext_lineitem; +DROP EXTERNAL TABLE ext_lineitem; + +--test 2.1 test writable table using compression with big data +CREATE WRITABLE EXTERNAL TABLE ext_lineitem_w ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE EXTERNAL TABLE ext_lineitem ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +INSERT INTO ext_lineitem_w SELECT * FROM lineitem; +DROP TABLE lineitem; +SELECT count(*) FROM ext_lineitem; +DROP EXTERNAL TABLE ext_lineitem_w; DROP EXTERNAL TABLE ext_lineitem; -- test 3 line too long with defaults @@ -193,7 +318,6 @@ FORMAT 'text' ; SELECT count(*) FROM ext_lineitem; DROP EXTERNAL TABLE ext_lineitem; - -- test 2 use a bigger file. CREATE EXTERNAL TABLE ext_lineitem ( @@ -247,4 +371,4 @@ DROP EXTERNAL TABLE ext_test; --test 4 using csv data CREATE EXTERNAL TABLE ext_crlf_with_lf_column(c1 int, c2 text) LOCATION ('gpfdist://@hostname@:7070/gpfdist2/crlf_with_lf_column.csv') FORMAT 'csv' (NEWLINE 'CRLF'); SELECT count(*) FROM ext_crlf_with_lf_column; -DROP EXTERNAL TABLE ext_crlf_with_lf_column; \ No newline at end of file +DROP EXTERNAL TABLE ext_crlf_with_lf_column; diff --git a/src/bin/gpfdist/regress/output/gpfdist2_compress.source b/src/bin/gpfdist/regress/output/gpfdist2_compress.source index 16d9a0d7cb87..a14f9c7a16d8 100644 --- a/src/bin/gpfdist/regress/output/gpfdist2_compress.source +++ b/src/bin/gpfdist/regress/output/gpfdist2_compress.source @@ -61,12 +61,80 @@ FORMAT 'text' DELIMITER AS '|' ) ; +CREATE TABLE lineitem (like ext_lineitem); +NOTICE: table doesn't have 'DISTRIBUTED BY' clause, defaulting to distribution columns from LIKE table SELECT count(*) FROM ext_lineitem; count ------- 256 (1 row) +INSERT INTO lineitem SELECT * FROM ext_lineitem; +DROP EXTERNAL TABLE ext_lineitem; +--test 1.1 test writable table using compression +CREATE WRITABLE EXTERNAL TABLE ext_lineitem_w ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE EXTERNAL TABLE ext_lineitem ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +INSERT INTO ext_lineitem_w SELECT * FROM lineitem; +DROP TABLE lineitem; +SELECT count(*) FROM ext_lineitem; + count +------- + 256 +(1 row) + +DROP EXTERNAL TABLE ext_lineitem_w; DROP EXTERNAL TABLE ext_lineitem; -- test 2 use a bigger file. CREATE EXTERNAL TABLE ext_lineitem ( @@ -96,12 +164,79 @@ FORMAT 'text' DELIMITER AS '|' ) ; +CREATE TABLE lineitem (like ext_lineitem); SELECT count(*) FROM ext_lineitem; count -------- 100000 (1 row) +INSERT INTO lineitem SELECT * FROM ext_lineitem; +DROP EXTERNAL TABLE ext_lineitem; +--test 2.1 test writable table using compression with big data +CREATE WRITABLE EXTERNAL TABLE ext_lineitem_w ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +CREATE EXTERNAL TABLE ext_lineitem ( + L_ORDERKEY INT8, + L_PARTKEY INTEGER, + L_SUPPKEY INTEGER, + L_LINENUMBER integer, + L_QUANTITY decimal, + L_EXTENDEDPRICE decimal, + L_DISCOUNT decimal, + L_TAX decimal, + L_RETURNFLAG CHAR(1), + L_LINESTATUS CHAR(1), + L_SHIPDATE date, + L_COMMITDATE date, + L_RECEIPTDATE date, + L_SHIPINSTRUCT CHAR(25), + L_SHIPMODE CHAR(10), + L_COMMENT VARCHAR(44) + ) +LOCATION +( + 'gpfdist://@hostname@:7070/gpfdist2/lineitem.tbl.w' +) +FORMAT 'text' +( + DELIMITER AS '|' +) +; +INSERT INTO ext_lineitem_w SELECT * FROM lineitem; +DROP TABLE lineitem; +SELECT count(*) FROM ext_lineitem; + count +-------- + 100256 +(1 row) + +DROP EXTERNAL TABLE ext_lineitem_w; DROP EXTERNAL TABLE ext_lineitem; -- test 3 line too long with defaults CREATE EXTERNAL TABLE ext_test ( From 06d9ac0a2f09ab043ad45354dc457b1218fb49c1 Mon Sep 17 00:00:00 2001 From: HouLei Date: Mon, 6 Feb 2023 16:35:49 +0800 Subject: [PATCH 12/32] fix compiler warning (#14910) fix a compiler warning. --- src/bin/gpfdist/gpfdist.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bin/gpfdist/gpfdist.c b/src/bin/gpfdist/gpfdist.c index c68c8c68ea92..e8e7a264663d 100644 --- a/src/bin/gpfdist/gpfdist.c +++ b/src/bin/gpfdist/gpfdist.c @@ -4753,7 +4753,7 @@ static int decompress_data(request_t* r, zstd_buffer *in, zstd_buffer *out){ ZSTD_outBuffer obuf = {out->buf, out->size, out->pos}; if(!r->zstd_dctx) { - gwarning(stderr, "%s", "Out of memory when ZSTD_createDCtx"); + gwarning(NULL, "%s", "Out of memory when ZSTD_createDCtx"); return -1; } From 0b6982029d844c4954af3cb7b169ee65d714628d Mon Sep 17 00:00:00 2001 From: hyongtao-db <99629139+hyongtao-db@users.noreply.github.com> Date: Thu, 16 Feb 2023 13:32:53 +0800 Subject: [PATCH 13/32] inclusive terminology for Greenplum 7 in gpfdist (#14987) rename master to coordinator in gpfdist **Signed-off-by:** Yongtao Huang --- src/bin/gpfdist/doc/init | 4 ++-- src/bin/gpfdist/doc/start | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bin/gpfdist/doc/init b/src/bin/gpfdist/doc/init index e0b9e64f5e5c..c8a8e66e39a9 100644 --- a/src/bin/gpfdist/doc/init +++ b/src/bin/gpfdist/doc/init @@ -52,7 +52,7 @@ MA === 1. do initdb for all segment except standby QD -2. copy data drom master QD to standby QD - "scp -r %s %s:%s" % ( seginfo[master_qd_idx].dir, +2. copy data from coordinator QD to standby QD + "scp -r %s %s:%s" % ( seginfo[coordinator_qd_idx].dir, seginfo[standby_qd_idx].host, seginfo[standby_qd_idx].dir) diff --git a/src/bin/gpfdist/doc/start b/src/bin/gpfdist/doc/start index c72e6f490723..d300bca72691 100644 --- a/src/bin/gpfdist/doc/start +++ b/src/bin/gpfdist/doc/start @@ -20,12 +20,12 @@ waitflag = (wait ? "-w" : "") -Start master QD +Start coordinator QD ===================== 1. kill gpsyncmaster -2. start master QD +2. start coordinator QD waitflag = (wait ? "-w" : "") @@ -50,7 +50,7 @@ fi -Start Standby Sync master +Start Standby Sync coordinator ========================= $GPHOME/bin/gpsyncmaster -p %d -D %s -i >> %s.log 2>&1 &" % (pgport, datadir, datadir) From 7656296f3e915fb3a25edc7fef9af8023084a83f Mon Sep 17 00:00:00 2001 From: HouLei Date: Thu, 23 Feb 2023 22:51:50 -0800 Subject: [PATCH 14/32] Double gpfdist listening to one port In an extreme case, two gpfdist occupy one port(ipv4/ipv6). The reason for this problem includes two aspects. The first one is that 'bind' is not mutually exclusive. And another is that when listening to a port fail, gpfdist will try the same port with a different protocol(ipv4 or ipv6). So the PR fixes the problem by changing the handling for failed listening behavior. --- src/bin/gpfdist/gpfdist.c | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/bin/gpfdist/gpfdist.c b/src/bin/gpfdist/gpfdist.c index e8e7a264663d..1488324b4e7c 100644 --- a/src/bin/gpfdist/gpfdist.c +++ b/src/bin/gpfdist/gpfdist.c @@ -2668,7 +2668,20 @@ http_setup(void) opt.p, saved_errno, strerror(saved_errno)); - continue; + +#ifdef WIN32 + if ( 1 ) +#else + if ( errno == EADDRINUSE ) +#endif + { + create_failed = true; + break; + } + else + { + gwarning(NULL, "%s (errno=%d), port: %d",strerror(errno), errno, opt.p); + } } gcb.listen_socks[gcb.listen_sock_count++] = f; From 92fbc2ec5b81e1fbacdac53cd1d099319e50c025 Mon Sep 17 00:00:00 2001 From: Alexey Gordeev Date: Fri, 26 May 2023 08:10:50 +0500 Subject: [PATCH 15/32] Fix url_curl.c headers handling (#14976) Previously, header_callback(), used to get and parse HTTP headers, took only first status header from all header blocks to show it to user. The problem is that we can have multiple messages with multiple header blocks. In case of error, only first (successful) status is shown to user, which is not informative. This patch fixes header_callback(), which now process all status headers. --- src/backend/access/external/url_curl.c | 12 ++++++++++-- src/bin/gpfdist/regress/input/exttab1.source | 4 +++- src/bin/gpfdist/regress/output/exttab1.source | 4 ++++ 3 files changed, 17 insertions(+), 3 deletions(-) diff --git a/src/backend/access/external/url_curl.c b/src/backend/access/external/url_curl.c index 0e530170bdf5..67f6662d031b 100644 --- a/src/backend/access/external/url_curl.c +++ b/src/backend/access/external/url_curl.c @@ -320,14 +320,19 @@ header_callback(void *ptr_, size_t size, size_t nmemb, void *userp) Assert(size == 1); /* - * parse the http response line (code and message) from + * Parse the http response line (code and message) from * the http header that we get. Basically it's the whole * first line (e.g: "HTTP/1.0 400 time out"). We do this * in order to capture any error message that comes from * gpfdist, and later use it to report the error string in * check_response() to the database user. + * As we can get multiple header blocks for multiple HTTP + * messages, we need to parse all headers and get the last + * one. First header may contain only successfull status and + * no info about the error. */ - if (url->http_response == 0) + + if (len > 5 && *ptr == 'H' && 0 == strncmp("HTTP/", ptr, 5)) { int n = nmemb; char* p; @@ -343,6 +348,9 @@ header_callback(void *ptr_, size_t size, size_t nmemb, void *userp) if (n > 0 && (p[n-1] == '\r' || p[n-1] == '\n')) p[--n] = 0; + if (url->http_response) + pfree(url->http_response); + url->http_response = p; } } diff --git a/src/bin/gpfdist/regress/input/exttab1.source b/src/bin/gpfdist/regress/input/exttab1.source index 6611128817cb..e921868256f4 100644 --- a/src/bin/gpfdist/regress/input/exttab1.source +++ b/src/bin/gpfdist/regress/input/exttab1.source @@ -487,7 +487,9 @@ COPY wet_region FROM STDIN DELIMITER '|'; 3|EUROPE|ly final courts cajole furiously final excuse \. INSERT INTO wet_region VALUES(4,'MIDDLE EAST','uickly special'); - +-- Check error correctness, if trying to insert too long row +INSERT INTO wet_pos1(a) +SELECT string_agg(md5(random()::text), '') from generate_series(1,1100); -- -- Now use RET to see if data was exported correctly. -- NOTE: since we don't bother cleaning up the exported file, it may grow bigger diff --git a/src/bin/gpfdist/regress/output/exttab1.source b/src/bin/gpfdist/regress/output/exttab1.source index 5ba2512145d9..363311316d67 100644 --- a/src/bin/gpfdist/regress/output/exttab1.source +++ b/src/bin/gpfdist/regress/output/exttab1.source @@ -681,6 +681,10 @@ COPY reg_region FROM STDIN DELIMITER '|'; INSERT INTO wet_region SELECT * from reg_region; COPY wet_region FROM STDIN DELIMITER '|'; INSERT INTO wet_region VALUES(4,'MIDDLE EAST','uickly special'); +-- Check error correctness, if trying to insert too long row +INSERT INTO wet_pos1(a) +SELECT string_agg(md5(random()::text), '') from generate_series(1,1100); +ERROR: http response code 500 from gpfdist (gpfdist://@hostname@:7070/wet.out): HTTP/1.0 500 no complete data row found for writing -- -- Now use RET to see if data was exported correctly. -- NOTE: since we don't bother cleaning up the exported file, it may grow bigger From e96069b117ba5094f523d501bba3e3673bc93970 Mon Sep 17 00:00:00 2001 From: HouLei Date: Mon, 5 Jun 2023 14:12:23 +0800 Subject: [PATCH 16/32] Add multi-thread compression and transmission from gpfdist to gpdb7 (#14698) Compression is very useful for data transmission with low bandwidth and high efficiency. But compression is a computationally intensive task and may increase the cost of time in the compression stage. To improve the efficiency of data transmission with compression, multi-thread data compression and transmission are introduced in this PR. The feature makes each segment served by a single thread. In the main thread, requests are read and further parsed into specific data structures. The data is read into a buffer, then a sub-thread is created to compress data and send the compressed data. To avoid conflict between threads, each request can be only served by one sub-thread. What's more, all public resources are accessed by a main thread to avoid waiting caused by thread synchronization. To reduce the extra cost incurred by thread switch, we limit the max number of threads according to the number of CPU cores. After the benchmark test in a local development environment, we proved two promotions: gpfdist saves about 60%~70% bandwidth occupation when compression is open. multi-thread compression has more efficient transmission. If you want multi-thread transmission with compression, both flag '--compress' and '--multi_thread' is required. To gain more efficiency, we recommend setting the max-length flag (-m) to be 1M. Co-authored by: @water32 --- gpMgmt/doc/gpfdist_help | 24 + src/backend/access/external/url_curl.c | 12 + src/backend/utils/misc/fstream/gfile.c | 2 +- src/bin/gpfdist/gpfdist.c | 468 ++++++++++++++---- .../regress/data/gpfdist2/one_column.txt | 10 + .../regress/input/gpfdist2_compress.source | 102 ++++ .../regress/output/gpfdist2_compress.source | 118 +++++ 7 files changed, 641 insertions(+), 95 deletions(-) create mode 100644 src/bin/gpfdist/regress/data/gpfdist2/one_column.txt diff --git a/gpMgmt/doc/gpfdist_help b/gpMgmt/doc/gpfdist_help index 52b253675981..f14d59c26b41 100755 --- a/gpMgmt/doc/gpfdist_help +++ b/gpMgmt/doc/gpfdist_help @@ -9,6 +9,7 @@ SYNOPSIS gpfdist [-d ] [-p ] [-l ] [-t ] [-S] [-w