diff --git a/README.Ubuntu.bash b/README.Ubuntu.bash index d7ef1c45f096..2a383b9d8d31 100755 --- a/README.Ubuntu.bash +++ b/README.Ubuntu.bash @@ -1,7 +1,7 @@ #!/bin/bash -sudo apt-get update -sudo apt-get install -y \ +apt-get update +apt-get install -y \ bison \ ccache \ cmake \ @@ -10,15 +10,20 @@ sudo apt-get install -y \ git-core \ gcc \ g++ \ + llvm \ + clang \ inetutils-ping \ krb5-kdc \ krb5-admin-server \ libapr1-dev \ libbz2-dev \ + libuv1-dev \ libcurl4-gnutls-dev \ libevent-dev \ libkrb5-dev \ libpam-dev \ + libldap-common \ + libldap-dev \ libperl-dev \ libreadline-dev \ libssl-dev \ @@ -35,12 +40,12 @@ sudo apt-get install -y \ pkg-config \ python3-dev \ python3-pip \ + python3-psycopg2 \ python3-psutil \ - python3-pygresql \ python3-yaml \ zlib1g-dev -sudo tee -a /etc/sysctl.conf << EOF +tee -a /etc/sysctl.conf << EOF kernel.shmmax = 5000000000000 kernel.shmmni = 32768 kernel.shmall = 40000000000 @@ -58,10 +63,10 @@ vm.overcommit_memory = 2 vm.overcommit_ratio = 95 EOF -sudo sysctl -p +sysctl -p -sudo mkdir -p /etc/security/limits.d -sudo tee -a /etc/security/limits.d/90-greenplum.conf << EOF +mkdir -p /etc/security/limits.d +tee -a /etc/security/limits.d/90-greenplum.conf << EOF * soft nofile 1048576 * hard nofile 1048576 * soft nproc 1048576 diff --git a/arenadata/.env b/arenadata/.env new file mode 100644 index 000000000000..382c67eb7605 --- /dev/null +++ b/arenadata/.env @@ -0,0 +1 @@ +COMPOSE_HTTP_TIMEOUT=400 diff --git a/arenadata/Dockerfile.linter b/arenadata/Dockerfile.linter new file mode 100644 index 000000000000..24089aab3bc9 --- /dev/null +++ b/arenadata/Dockerfile.linter @@ -0,0 +1,12 @@ +FROM ubuntu:20.04 + +RUN apt update && apt install -y git parallel clang-format-11 && \ + ln -s /usr/bin/clang-format-11 /usr/bin/clang-format + +WORKDIR /opt/gpdb_src + +COPY . /opt/gpdb_src + +ENV CLANG_FORMAT=clang-format-11 + +ENTRYPOINT src/tools/fmt gen && git diff --exit-code && src/tools/fmt chk diff --git a/arenadata/Dockerfile.ubuntu b/arenadata/Dockerfile.ubuntu new file mode 100644 index 000000000000..048d684d224b --- /dev/null +++ b/arenadata/Dockerfile.ubuntu @@ -0,0 +1,54 @@ +FROM ubuntu:22.04 AS base + +COPY README.Ubuntu.bash ./ + +RUN set -eux; \ + export DEBIAN_FRONTEND=noninteractive; \ + ./README.Ubuntu.bash; \ + rm README.Ubuntu.bash; \ +# The en_US.UTF-8 locale is needed to run GPDB + locale-gen en_US.UTF-8; \ +# To run sshd directly, but not using `/etc/init.d/ssh start` + mkdir /run/sshd; \ +# Alter precedence in favor of IPv4 during resolving + echo 'precedence ::ffff:0:0/96 100' >> /etc/gai.conf; \ +# Packages for tests + apt install -y rsync iproute2 fakeroot lsof sudo python3.11 python3.11-dev openjdk-17-jdk libipc-run-perl; \ + apt install -y protobuf-compiler libpq-dev libossp-uuid-dev; \ +# Install allure-behave for behave tests + python3 -m pip install psutil; \ + pip3 install pytest gsutil behave~=1.2.6 coverage~=4.5 'mock<=5.0.0' allure-behave==2.4.0 protobuf==3.20.0; \ + pip3 install PyGreSQL; \ + pip3 cache purge + +WORKDIR /home/gpadmin + +RUN mkdir -p /gpadmin/.ssh/ && cat >> /gpadmin/.ssh/config <<-EOF +Host * + StrictHostKeyChecking no +EOF + +FROM base AS build + +COPY . gpdb_src + +RUN mkdir /home/gpadmin/bin_gpdb + +ENV TARGET_OS_VERSION=22 +ENV TARGET_OS=ubuntu +ENV OUTPUT_ARTIFACT_DIR=bin_gpdb +ENV SKIP_UNITTESTS= +# To set default locale for unittests +ENV LANG=en_US.UTF-8 +ENV CONFIGURE_FLAGS="--enable-debug-extensions --with-gssapi --with-openssl --enable-cassert --enable-debug --enable-depend --with-libxml --with-uuid=ossp --with-perl --enable-mapreduce --enable-orafce" + +# Compile with running mocking tests +RUN bash /home/gpadmin/gpdb_src/concourse/scripts/compile_gpdb.bash + +FROM base AS code +COPY . gpdb_src +RUN rm -rf gpdb_src/.git/ + +FROM base AS test +COPY --from=code /home/gpadmin/gpdb_src gpdb_src +COPY --from=build /home/gpadmin/bin_gpdb /home/gpadmin/bin_gpdb diff --git a/arenadata/docker-compose.yaml b/arenadata/docker-compose.yaml new file mode 100644 index 000000000000..807d896aa8f0 --- /dev/null +++ b/arenadata/docker-compose.yaml @@ -0,0 +1,56 @@ +--- + +version: "3" +services: + #image can be either hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}-x86-64 or + #hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}-ppc64le depending on the arch + cdw: + image: "${IMAGE}" + working_dir: /home/gpadmin + hostname: cdw + volumes: + - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa" + - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub" + - "$PWD/allure-results:/tmp/allure-results" + privileged: true + sysctls: + kernel.sem: 500 1024000 200 4096 + init: true + entrypoint: > + sleep infinity + sdw1: + image: "${IMAGE}" + privileged: true + hostname: sdw1 + volumes: + - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa" + - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub" + sysctls: + kernel.sem: 500 1024000 200 4096 + init: true + entrypoint: > + sleep infinity + sdw2: + image: "${IMAGE}" + privileged: true + hostname: sdw2 + volumes: + - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa" + - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub" + sysctls: + kernel.sem: 500 1024000 200 4096 + init: true + entrypoint: > + sleep infinity + sdw3: + image: "${IMAGE}" + privileged: true + hostname: sdw3 + volumes: + - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa" + - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub" + sysctls: + kernel.sem: 500 1024000 200 4096 + init: true + entrypoint: > + sleep infinity diff --git a/arenadata/scripts/behave_gpdb.bash b/arenadata/scripts/behave_gpdb.bash new file mode 100755 index 000000000000..36b8b58b8e74 --- /dev/null +++ b/arenadata/scripts/behave_gpdb.bash @@ -0,0 +1,52 @@ +#!/bin/bash -l + +set -eox pipefail + +CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../concourse/scripts" && pwd )" +source "${CWDIR}/common.bash" + +function gen_env(){ + cat > /opt/run_test.sh <<-EOF + set -ex + + source /usr/local/greenplum-db-devel/greenplum_path.sh + + source gpdb_src/gpAux/gpdemo/gpdemo-env.sh + + if [[ ${FEATURE} == "gpexpand" ]]; then + mkdir -p /home/gpadmin/sqldump + wget -nv https://rt.adsw.io/artifactory/common/dump.sql.xz -O /home/gpadmin/sqldump/dump.sql.xz + + xz -d /home/gpadmin/sqldump/dump.sql.xz + fi + + cd "\${1}/gpdb_src/gpMgmt/" + BEHAVE_TAGS="${BEHAVE_TAGS}" + BEHAVE_FLAGS="${BEHAVE_FLAGS}" + if [ ! -z "\${BEHAVE_TAGS}" ]; then + make -f Makefile.behave behave tags=\${BEHAVE_TAGS} + else + flags="\${BEHAVE_FLAGS}" make -f Makefile.behave behave + fi + EOF + + chmod a+x /opt/run_test.sh +} + +function _main() { + + if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then + echo "FATAL: BEHAVE_TAGS or BEHAVE_FLAGS not set" + exit 1 + fi + + # Run inside a subshell so it does not pollute the environment after + # sourcing greenplum_path + time (make_cluster) + + time gen_env + + time run_test +} + +_main "$@" diff --git a/arenadata/scripts/run_behave_tests.bash b/arenadata/scripts/run_behave_tests.bash new file mode 100755 index 000000000000..34b38d5ddf9b --- /dev/null +++ b/arenadata/scripts/run_behave_tests.bash @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +set -x -o pipefail + +behave_tests_dir="gpMgmt/test/behave/mgmt_utils" + +# TODO concourse_cluster tests are not stable +# clusters="concourse_cluster ~concourse_cluster,demo_cluster" + +clusters="~concourse_cluster" + +if [ $# -eq 0 ] +then + # TODO cross_subnet and gpssh tests are excluded + features=`ls $behave_tests_dir -1 | grep feature | grep -v -E "cross_subnet|gpssh" | sed 's/\.feature$//'` +else + for feature in $@ + do + if [ ! -f "$behave_tests_dir/$feature.feature" ] + then + echo "Feature '$feature' doesn't exists" + exit 1 + fi + done + features=$@ +fi + +processes=3 + +rm -rf allure-results +mkdir allure-results -pm 777 +mkdir ssh_keys -p +if [ ! -e "ssh_keys/id_rsa" ] +then + ssh-keygen -P "" -f ssh_keys/id_rsa +fi + +run_feature() { + local feature=$1 + local cluster=$2 + if [ $cluster = "concourse_cluster" ]; then + local project="${feature}_concourse" + else + local project="${feature}_demo" + fi + echo "Started $feature behave tests on cluster $cluster and project $project" + docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env up -d + # Prepare ALL containers first + local services=$(docker-compose -p $project -f arenadata/docker-compose.yaml config --services | tr '\n' ' ') + for service in $services + do + docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \ + $service bash -c "mkdir -p /data/gpdata && chmod -R 777 /data && + source gpdb_src/concourse/scripts/common.bash && install_gpdb && + ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash" & + done + wait + + # Add host keys to known_hosts after containers setup + for service in $services + do + docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \ + $service bash -c "ssh-keyscan ${services/$service/} >> /home/gpadmin/.ssh/known_hosts" & + done + wait + + docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \ + -e FEATURE="$feature" -e BEHAVE_FLAGS="--tags $feature --tags=$cluster \ + -f behave_utils.arenadata.formatter:CustomFormatter \ + -o non-existed-output \ + -f allure_behave.formatter:AllureFormatter \ + -o /tmp/allure-results" \ + cdw gpdb_src/arenadata/scripts/behave_gpdb.bash + status=$? + + docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env down -v + + if [[ $status > 0 ]]; then echo "Feature $feature failed with exit code $status"; fi + exit $status +} + +pids="" +exits=0 +for feature in $features +do + for cluster in $clusters + do + run_feature $feature $cluster & + pids+="$! " + if [[ $(jobs -r -p | wc -l) -ge $processes ]]; then + wait -n + ((exits += $?)) + fi + done +done +for pid in $pids + do + wait $pid + exits=$((exits + $?)) + done +if [[ $exits > 0 ]]; then exit 1; fi diff --git a/concourse/pipelines/gen_pipeline.py b/concourse/pipelines/gen_pipeline.py index 83632b9ea9c2..ec3d528fc5b2 100755 --- a/concourse/pipelines/gen_pipeline.py +++ b/concourse/pipelines/gen_pipeline.py @@ -153,12 +153,51 @@ def create_pipeline(args, git_remote, git_branch): test_trigger = "false" variables_type = args.pipeline_target + os_username = { + "rhel8" : "rhel", + "rocky8" : "rocky", + "oel8" : "oel", + "rhel9" : "rhel", + "rocky9" : "rocky", + "oel9" : "oel" + } + test_os = { + "rhel8" : "centos", + "rocky8": "centos", + "oel8" : "centos", + "rhel9" : "centos", + "rocky9": "centos", + "oel9" : "centos" + } + compile_platform = { + "rhel8" : "rocky8", + "rocky8": "rocky8", + "oel8" : "rocky8", + "rhel9" : "rocky9", + "rocky9": "rocky9", + "oel9" : "rocky9" + } + dist = { + "rhel8" : "el8", + "rocky8" : "el8", + "oel8" : "el8", + "rhel9" : "el9", + "rocky9" : "el9", + "oel9": "el9" + } + context = { 'template_filename': args.template_filename, 'generator_filename': os.path.basename(__file__), 'timestamp': datetime.datetime.now(), - 'os_types': args.os_types, + 'os_type': args.os_type, + 'default_os_type': default_os_type, + 'os_username': os_username[args.os_type], + 'test_os': test_os[args.os_type], + 'compile_platform': compile_platform[args.os_type], + 'dist': dist[args.os_type], + 'pipeline_target': args.pipeline_target, 'test_sections': args.test_sections, 'pipeline_configuration': args.pipeline_configuration, 'test_trigger': test_trigger, @@ -397,7 +436,11 @@ def main(): exit(1) if args.pipeline_target == 'prod' and not args.directed_release: - args.pipeline_configuration = 'prod' + args.test_sections = [ + 'icw', + 'cli', + 'release' + ] # use_ICW_workers adds tags to the specified concourse definitions which # correspond to dedicated concourse workers to increase performance. diff --git a/concourse/pipelines/gpdb_main-generated.yml b/concourse/pipelines/gpdb_main-generated.yml index b82930522fa9..78b5c45cfc20 100644 --- a/concourse/pipelines/gpdb_main-generated.yml +++ b/concourse/pipelines/gpdb_main-generated.yml @@ -12,10 +12,10 @@ ## file (example: templates/gpdb-tpl.yml) and regenerate the pipeline ## using appropriate tool (example: gen_pipeline.py -t prod). ## ---------------------------------------------------------------------- -## Generated by gen_pipeline.py at: 2022-11-14 22:19:02.574098 +## Generated by gen_pipeline.py at: 2023-01-11 15:47:56.669386 ## Template file: gpdb-tpl.yml -## OS Types: ['rhel8', 'win'] -## Test Sections: ['ICW', 'Replication', 'ResourceGroups', 'Interconnect', 'CLI', 'UD', 'Extensions'] +## OS Types: +## Test Sections: ['icw', 'cli', 'release'] ## ====================================================================== ## ====================================================================== @@ -931,6 +931,45 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: test_upgrade + file: gpdb_src/concourse/tasks/test_upgrade.yml + input_mapping: + sqldump: icw_planner_rocky8_dump + params: + NUMBER_OF_NODES: 2 + +- name: icw_planner_resgroup_rocky8 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_icw_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-32 + number_of_nodes: 1 + segments_per_host: 3 + disk_size: 800 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + DATA_MOUNT_DIR: "/home/gpadmin" + CREATE_DATA_DIR: false - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: ccp-image @@ -959,9 +998,11 @@ jobs: passed: - compile_gpdb_rhel8 trigger: true - - get: bin_gpdb_rhel8 + - get: bin_gpdb + params: + skip_download: 'true' passed: - - compile_gpdb_rhel8 + - gate_compile_end - name: check_centos plan: @@ -1031,6 +1072,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpmovemirrors_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1229,6 +1272,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpstate_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1315,6 +1360,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpactivatestandby_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1375,6 +1422,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpinitstandby_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1521,6 +1570,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpcheckperf_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1581,6 +1632,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gprecoverseg_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1658,12 +1711,14 @@ jobs: head -n -2 cluster_env_files/hostfile_init_orig > cluster_env_files/hostfile_init - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gprecoverseg_newhost_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test params: SUDO_ACCESS_ON_HOSTS: 1 - SUDO_MDW_USERNAME: rhel + SUDO_CDW_USERNAME: rhel BEHAVE_FLAGS: --tags=gprecoverseg_newhost --tags=concourse_cluster on_success: @@ -1720,6 +1775,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpaddmirrors_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1780,6 +1837,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpconfig_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1840,6 +1899,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpssh-exkeys_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1900,6 +1961,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpstart_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1960,6 +2023,8 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: gpstop_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test @@ -1984,6 +2049,85 @@ jobs: BEHAVE_FLAGS: --tags=gpstop --tags=~concourse_cluster,demo_cluster +- name: resource_group_rocky8 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml + image: ccp-image + params: + TEST_OS: rocky8 + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +- name: resource_group_v2_rocky8 + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + PLATFORM: rocky8-cgroup2-gpdb7 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: rocky8-cgroup2-gpdb7 + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup_v2.yml + image: ccp-image + params: + TEST_OS: rocky8 + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed - name: cli_cross_subnet plan: @@ -2109,11 +2253,12 @@ jobs: cp cluster_2/.ssh/*.pem cluster_env_files/.ssh cat cluster_2/.ssh/known_hosts >> cluster_env_files/.ssh/known_hosts cat cluster_2/.ssh/config >> cluster_env_files/.ssh/config - sed -i '/^Host mdw-1/ s/$/ mdw/' cluster_env_files/.ssh/config + sed -i '/^Host cdw-1/ s/$/ cdw/' cluster_env_files/.ssh/config - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml params: - GPINIT_STANDBY_MASTER: '-s mdw-2' + GPINIT_STANDBY_COORDINATOR: '-s cdw-2' + <<: *default_platform - task: cross_subnet_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: ccp-image @@ -2267,7 +2412,7 @@ jobs: <<: *set_failed - # end CLI test_sections + # end cli test_sections ## ====================================================================== ## ____ _ @@ -2452,15 +2597,15 @@ jobs: RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) - in_parallel: steps: - - put: bin_gpdb_rhel8_rc - params: - file: "release_candidates/server-rc-*rhel8*.tar.gz" - - put: bin_gpdb_clients_rhel8_rc - params: - file: "release_candidates/clients-rc-*rhel8*.tar.gz" - put: server_src_rc params: file: "server_tar/*server-src-rc-*.tar.gz" + - put: bin_gpdb_rc + params: + file: "release_candidates/server-rc-*el8*.tar.gz" + - put: bin_gpdb_clients_rc + params: + file: "release_candidates/clients-rc-*el8*.tar.gz" - name: Build_Release_Candidate_RPMs diff --git a/concourse/pipelines/templates/gpdb-tpl.yml b/concourse/pipelines/templates/gpdb-tpl.yml index a87e5aa1df10..a8cc77550a7f 100644 --- a/concourse/pipelines/templates/gpdb-tpl.yml +++ b/concourse/pipelines/templates/gpdb-tpl.yml @@ -203,6 +203,8 @@ anchors: groups: - name: all jobs: + - gate_compile_start + {% if os_type == compile_platform %} - check_format {% if "rhel8" in os_types %} - compile_gpdb_rhel8 @@ -264,17 +266,21 @@ groups: {%endif %} -{% if pipeline_configuration == "prod" or build_test_rc_rpm or directed_release %} +{% if ( ("release" in test_sections and os_type == compile_platform) or build_test_rc_rpm or directed_release) %} ## ====================================================================== - name: Release jobs: -{% if pipeline_configuration == "prod" or directed_release %} +{% if "release" in test_sections or directed_release %} + - gate_icw_end + - gate_cli_end - gate_release_candidate_start - Release_Candidate {% if not directed_release %} - Publish Server Builds {% endif %} +{% if not directed_release %} + - build_release_candidate_rpms {% endif %} {% if not directed_release %} - Build_Release_Candidate_RPMs @@ -285,11 +291,13 @@ groups: - name: Compile jobs: -{% if "rhel8" in os_types %} - - compile_gpdb_rhel8 - - test_gpdb_clients_rhel8 -{% endif %} -{% if "win" in os_types %} + - gate_compile_start + - compile_gpdb_[[ compile_platform ]] + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - prepare_binary_swap_gpdb_[[ os_type ]] + {% endif %} + - test_gpdb_clients_[[ os_type ]] + {% if os_type == compile_platform %} - compile_gpdb_clients_windows {% endif %} @@ -470,7 +478,17 @@ resources: username: _json_key password: ((data-gpdb-private-images-container-registry-readonly-service-account-key)) -- name: gpdb7-rhel8-test +{% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} +- name: previous_gpdb + type: tanzunet + source: + api_token: ((pivotal-gpdb-pivnet-api-token)) + product_slug: pivotal-gpdb + product_version: ((pivotal-gpdb-pivnet-product-version)) +{% endif %} + +{% if os_type != compile_platform %} +- name: gpdb7-[[ compile_platform ]]-build type: registry-image source: repository: gcr.io/data-gpdb-private-images/gpdb7-rhel8-test @@ -481,7 +499,7 @@ resources: {% if pipeline_configuration == "prod" or directed_release %} {% if not directed_release %} -- name: bin_gpdb_rhel8_icw_green +- name: bin_gpdb_icw_green type: s3 source: access_key_id: ((aws-bucket-access-key-id)) @@ -490,20 +508,20 @@ resources: secret_access_key: ((aws-bucket-secret-access-key)) versioned_file: bin_gpdb_rhel8/gpdb_branch_master/icw_green/bin_gpdb.tar.gz -- name: bin_gpdb_rhel8_rc +{% if os_type == compile_platform %} +- name: bin_gpdb_rc type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/server-rc-(.*)-rhel8_x86_64((rc-build-type-gcs)).tar.gz + regexp: server/published/main/server-rc-(.*)-[[ dist ]]_x86_64((rc-build-type-gcs)).tar.gz -- name: bin_gpdb_clients_rhel8_rc +- name: bin_gpdb_clients_rc type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: clients/published/main/clients-rc-(.*)-rhel8_x86_64((rc-build-type-gcs)).tar.gz - + regexp: clients/published/main/clients-rc-(.*)-[[ dist ]]_x86_64((rc-build-type-gcs)).tar.gz {% endif %} - name: server_src_rc type: gcs @@ -517,13 +535,13 @@ resources: {% endif %} {% endif %} -{% if pipeline_configuration == "prod" or build_test_rc_rpm %} -- name: gpdb_rpm_installer_rhel8 +{% if ( ("release" in test_sections and os_type == compile_platform) or build_test_rc_rpm) %} +- name: gpdb_rpm_installer_[[ os_type ]] type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/greenplum-db-(.*)-rhel8-x86_64((rc-build-type-gcs)).rpm + regexp: server/published/main/greenplum-db-(.*)-[[ dist ]]-x86_64((rc-build-type-gcs)).rpm - name: greenplum-database-release type: git @@ -549,25 +567,33 @@ resources: json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_rhel8/bin_gpdb.tar.gz -- name: bin_gpdb_clients_rhel8 +{% if compile_platform == default_os_type %} +- name: llvm-with-asserts-packages + type: gcs + source: + bucket: pivotal-gpdb-concourse-resources-prod + json_key: ((concourse-gcs-resources-service-account-key)) + regexp: gp-internal-artifacts/[[ default_os_type ]]/llvm-with-asserts-(.*)-[[ dist ]]_x86_64.tar.gz +{% endif %} + +{% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} +- name: binary_swap_gpdb_[[ os_type ]] type: gcs source: bucket: ((gcs-bucket-intermediates)) json_key: ((concourse-gcs-resources-service-account-key)) versioned_file: ((pipeline-name))/bin_gpdb_clients_rhel8/bin_gpdb_clients.tar.gz -{% if pipeline_configuration == "prod" %} -- name: server-build-rhel8 +{% if os_type == compile_platform %} +- name: server-build-[[ os_type ]] type: gcs source: bucket: ((gcs-bucket)) json_key: ((concourse-gcs-resources-service-account-key)) - regexp: server/published/main/server-build-(.*)-rhel8_x86_64((rc-build-type-gcs)).tar.gz -{% endif %} - + regexp: server/published/main/server-build-(.*)-[[ dist ]]_x86_64((rc-build-type-gcs)).tar.gz {% endif %} -{% if "win" in os_types %} +{% if os_type == compile_platform %} - name: bin_gpdb_clients_windows type: gcs source: @@ -606,6 +632,31 @@ jobs: ## |_| ## ====================================================================== +## ====================================================================== +## ____ _ _ +## / ___|___ _ __ ___ _ __ (_) | ___ +## | | / _ \| '_ ` _ \| '_ \| | |/ _ \ +## | |__| (_) | | | | | | |_) | | | __/ +## \____\___/|_| |_| |_| .__/|_|_|\___| +## |_| +## ====================================================================== + +- name: gate_compile_start + plan: + - get: gpdb_src + {% if os_type != compile_platform and pipeline_target == "prod" %} + trigger: false + {% else %} + trigger: true + {% endif %} + params: + skip_download: 'true' + {% if os_type != compile_platform and pipeline_target == "prod" %} + - get: reduced-frequency-trigger + trigger: true + {% endif %} + +{% if os_type == compile_platform %} - name: check_format plan: - get: gpdb_src @@ -637,9 +688,16 @@ jobs: trigger: ((reduced-frequency-trigger-flag)) - get: gpdb_src passed: + {% if os_type == compile_platform %} - check_format - trigger: ((gpdb_src-trigger-flag)) - - get: gpdb7-rhel8-build + {% else %} + - gate_compile_start + {% endif %} + trigger: true + - get: gpdb7-[[ compile_platform ]]-build + {% if compile_platform == default_os_type %} + - get: llvm-with-asserts-packages + {% endif %} - task: compile_gpdb image: gpdb7-rhel8-build file: gpdb_src/concourse/tasks/compile_gpdb.yml @@ -656,7 +714,28 @@ jobs: params: file: gpdb_artifacts/bin_gpdb_clients.tar.gz -- name: test_gpdb_clients_rhel8 +{% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} +- name: prepare_binary_swap_gpdb_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + trigger: true + passed: [compile_gpdb_[[ compile_platform ]]] + - get: gpdb_package + resource: previous_gpdb + params: + globs: [greenplum-db-7*-el8-x86_64.rpm] + - get: gpdb7-[[ compile_platform ]]-build + - task: generate_previous_bin_gpdb + file: gpdb_src/concourse/tasks/extract_package.yml + image: gpdb7-[[ compile_platform ]]-build + - put: binary_swap_gpdb_[[ os_type ]] + params: + file: gpdb_artifacts/bin_gpdb.tar.gz +{% endif %} + +- name: test_gpdb_clients_[[ os_type ]] plan: - in_parallel: steps: @@ -674,8 +753,7 @@ jobs: image: gpdb7-rhel8-test file: gpdb_src/concourse/tasks/test_gpdb_clients.yml -{% endif %} -{% if "win" in os_types %} +{% if os_type == compile_platform %} - name: compile_gpdb_clients_windows serial: true plan: @@ -683,10 +761,12 @@ jobs: steps: - get: gpdb_src trigger: true - - get: gpdb7-rhel8-build + passed: + - check_format + - get: gpdb7-[[ compile_platform ]]-build - task: compile_gpdb_windows_remote file: gpdb_src/concourse/tasks/compile_gpdb_remote_windows.yml - image: gpdb7-rhel8-build + image: gpdb7-[[ compile_platform ]]-build params: REMOTE_HOST: ((remote_win_host_build)) REMOTE_PORT: ((remote_win_port_build)) @@ -695,8 +775,92 @@ jobs: - put: bin_gpdb_clients_windows params: file: "gpdb_artifacts/greenplum-clients-x86_64.tar.gz" + +- name: publish_server_builds + old_name: Publish Server Builds + plan: + - in_parallel: + steps: + - get: gpdb_src + trigger: true + passed: + - compile_gpdb_[[ compile_platform ]] + - get: bin_gpdb + passed: [compile_gpdb_[[ compile_platform ]]] + - get: gpdb7-[[ compile_platform ]]-build + - task: rename_server_build_artifacts + image: gpdb7-[[ compile_platform ]]-build + config: + platform: linux + inputs: + - name: gpdb_src + - name: bin_gpdb + outputs: + - name: output + params: + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) + run: + path: bash + args: + - -ec + - | + server_version="$(./gpdb_src/getversion --short)" + cp bin_gpdb/bin_gpdb.tar.gz "output/server-build-${server_version}-[[ dist ]]_x86_64${RC_BUILD_TYPE_GCS}.tar.gz" + - in_parallel: + steps: + - put: server-build-[[ os_type ]] + params: + file: output/server-build-*-[[ dist ]]*.tar.gz + {% endif %} -{% if "ICW" in test_sections %} + +- name: gate_compile_end + plan: + - get: gpdb_src + trigger: true + params: + skip_download: 'true' + passed: + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - prepare_binary_swap_gpdb_[[ os_type ]] + {% endif %} + {% if os_type == compile_platform %} + - compile_gpdb_clients_windows + - publish_server_builds + {% endif %} + - test_gpdb_clients_[[ os_type ]] + - get: bin_gpdb + params: + skip_download: 'true' + passed: + - test_gpdb_clients_[[ os_type ]] + {% if os_type == compile_platform %} + - publish_server_builds + {% endif %} + - get: bin_gpdb_clients + params: + skip_download: 'true' + passed: + - test_gpdb_clients_[[ os_type ]] + {% if os_type == compile_platform %} + - get: bin_gpdb_clients_windows + params: + skip_download: 'true' + passed: + - compile_gpdb_clients_windows + - get: server-build-[[ os_type ]] + params: + skip_download: 'true' + passed: [publish_server_builds] + {% endif %} + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb_[[ os_type ]] + params: + skip_download: 'true' + passed: [ prepare_binary_swap_gpdb_[[ os_type ]] ] + {% endif %} + +{% if "icw" in test_sections %} ## ====================================================================== ## ___ ______ __ ## |_ _/ ___\ \ / / @@ -704,8 +868,31 @@ jobs: ## | | |___ \ V V / ## |___\____| \_/\_/ ## ====================================================================== -{% if "rhel8" in os_types %} -- name: gpdb_pitr_rhel8 +- name: gate_icw_start + plan: + - in_parallel: + steps: + - get: bin_gpdb + params: + skip_download: 'true' + passed: [gate_compile_end] + trigger: true + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb_[[ os_type ]] + params: + skip_download: 'true' + passed: [gate_compile_end] + {% endif %} + - get: gpdb_src + params: + skip_download: 'true' + passed: [gate_compile_end] + - get: bin_gpdb_clients + params: + skip_download: 'true' + passed: [ gate_compile_end ] + +- name: gpdb_pitr_[[ os_type ]] plan: - in_parallel: steps: @@ -785,12 +972,50 @@ jobs: tags: [icw-rhel8] {% endif %} file: gpdb_src/concourse/tasks/ic_gpdb.yml - image: gpdb7-rhel8-test + image: gpdb7-[[ os_type ]]-test + timeout: 24h + params: + MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=on -c jit=off' installcheck-world + TEST_OS: [[ test_os ]] + CONFIGURE_FLAGS: ((configure_flags)) + BLDWRAP_POSTGRES_CONF_ADDONS: + - optimizer_use_gpdb_allocators=off + +- name: icw_planner_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: bin_gpdb + passed: [gate_icw_start] + trigger: true + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb + passed: [gate_icw_start] + resource: binary_swap_gpdb_[[ os_type ]] + {% endif %} + - get: gpdb7-[[ os_type ]]-test + {% if compile_platform == default_os_type %} + - get: llvm-with-asserts-packages + {% endif %} + - task: ic_gpdb + {% if use_ICW_workers %} + tags: [icw-worker] + {% endif %} + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + file: gpdb_src/concourse/tasks/ic_gpdb_binary_swap.yml + {% else %} + file: gpdb_src/concourse/tasks/ic_gpdb.yml + {% endif %} + image: gpdb7-[[ os_type ]]-test timeout: 24h params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off -c jit=off' installcheck-world - TEST_OS: centos + TEST_OS: [[ test_os ]] + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} TEST_BINARY_SWAP: ((test-binary-swap)) + {% endif %} CONFIGURE_FLAGS: ((configure_flags)) DUMP_DB: "true" - put: icw_planner_rhel8_dump @@ -804,20 +1029,34 @@ jobs: - get: gpdb_src passed: [compile_gpdb_rhel8] - get: bin_gpdb - passed: [compile_gpdb_rhel8] - resource: bin_gpdb_rhel8 - trigger: [[ test_trigger ]] - - get: gpdb7-rhel8-test + passed: [gate_icw_start] + trigger: true + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + - get: binary_swap_gpdb + passed: [gate_icw_start] + resource: binary_swap_gpdb_[[ os_type ]] + {% endif %} + - get: gpdb7-[[ os_type ]]-test + {% if compile_platform == default_os_type %} + - get: llvm-with-asserts-packages + {% endif %} - task: ic_gpdb {% if use_ICW_workers %} tags: [icw-rhel8] {% endif %} + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + file: gpdb_src/concourse/tasks/ic_gpdb_binary_swap.yml + {% else %} file: gpdb_src/concourse/tasks/ic_gpdb.yml - image: gpdb7-rhel8-test + {% endif %} + image: gpdb7-[[ os_type ]]-test timeout: 24h params: MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off -c jit=on -c jit_above_cost=0 -c gp_explain_jit=off' installcheck - TEST_OS: centos + TEST_OS: [[ test_os ]] + {% if os_type == "rhel8" or os_type == "oel8" or os_type == "rocky8" %} + TEST_BINARY_SWAP: ((test-binary-swap)) + {% endif %} CONFIGURE_FLAGS: ((configure_flags)) - name: icw_gporca_icproxy_rhel8 @@ -1063,6 +1302,88 @@ jobs: PLATFORM: rhel8-gpdb7 - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: test_upgrade + file: gpdb_src/concourse/tasks/test_upgrade.yml + input_mapping: + sqldump: icw_planner_[[ os_type ]]_dump + params: + NUMBER_OF_NODES: 2 + +- name: icw_planner_resgroup_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_icw_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-32 + number_of_nodes: 1 + segments_per_host: 3 + disk_size: 800 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + DATA_MOUNT_DIR: "/home/gpadmin" + CREATE_DATA_DIR: false + - task: run_tests + file: gpdb_src/concourse/tasks/ic_resgroup.yml + image: ccp-image + params: + TEST_OS: [[ os_type ]] + MAKE_TEST_COMMAND: -k PGOPTIONS='-c optimizer=off -c jit=off' installcheck-world + BLDWRAP_POSTGRES_CONF_ADDONS: + - gp_resource_manager=group + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +- name: icw_gporca_resgroup_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_icw_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_icw_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-32 + number_of_nodes: 1 + segments_per_host: 3 + disk_size: 800 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + DATA_MOUNT_DIR: "/home/gpadmin" + CREATE_DATA_DIR: false - task: run_tests file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml image: ccp-image @@ -1073,6 +1394,61 @@ jobs: ensure: <<: *set_failed +- name: gate_icw_end + plan: + - in_parallel: + steps: + - get: bin_gpdb + passed: + - icw_gporca_[[ os_type ]] + - icw_gporca_jit_[[ os_type ]] + - icw_gporca_memory_[[ os_type ]] + - icw_planner_jit_[[ os_type ]] + - icw_gporca_icproxy_[[ os_type ]] + - icw_planner_icproxy_[[ os_type ]] + - icw_gporca_ictcp_[[ os_type ]] + - icw_mirrorless_[[ os_type ]] + - icw_extensions_gpcloud_[[ os_type ]] + - unit_tests_gporca_[[ os_type ]] + - gpdb_pitr_[[ os_type ]] + - gpexpand_[[ os_type ]] + - pg_upgrade_[[ os_type ]] + - interconnect_[[ os_type ]] + - get: icw_planner_[[ os_type ]]_dump + params: + skip_download: 'true' + passed: + - pg_upgrade_[[ os_type ]] + - gpexpand_[[ os_type ]] + - get: gpdb_src + params: + skip_download: 'true' + passed: + - icw_gporca_[[ os_type ]] + - icw_gporca_jit_[[ os_type ]] + - icw_gporca_memory_[[ os_type ]] + - icw_planner_jit_[[ os_type ]] + - icw_gporca_icproxy_[[ os_type ]] + - icw_planner_icproxy_[[ os_type ]] + - icw_gporca_ictcp_[[ os_type ]] + - icw_mirrorless_[[ os_type ]] + - icw_extensions_gpcloud_[[ os_type ]] + - unit_tests_gporca_[[ os_type ]] + - gpdb_pitr_[[ os_type ]] + - gpexpand_[[ os_type ]] + - pg_upgrade_[[ os_type ]] + - icw_planner_resgroup_[[ os_type ]] + - icw_gporca_resgroup_[[ os_type ]] + - interconnect_[[ os_type ]] + trigger: true + - get: bin_gpdb_clients + params: + skip_download: 'true' + passed: [unit_tests_gporca_[[ os_type ]]] +{% if ("release" in test_sections and os_type == compile_platform) %} + - put: bin_gpdb_icw_green + params: + file: bin_gpdb/bin_gpdb.tar.gz {% endif %} {% endif %} @@ -1096,7 +1472,7 @@ jobs: trigger: true - get: bin_gpdb_rhel8 passed: - - compile_gpdb_rhel8 + - gate_compile_end - name: check_centos plan: @@ -1188,13 +1564,15 @@ jobs: {% endif %} - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform - task: [[ test.name ]]_concourse_cluster_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: gpdb7-rhel8-test params: {% if test.sudo_access_on_hosts %} SUDO_ACCESS_ON_HOSTS: 1 - SUDO_MDW_USERNAME: rhel + SUDO_CDW_USERNAME: rhel {% endif %} BEHAVE_FLAGS: --tags=[[ test.name ]] --tags=concourse_cluster [[ test.env ]] @@ -1220,6 +1598,86 @@ jobs: {% endif %} {% endfor %} +- name: resource_group_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup.yml + image: ccp-image + params: + TEST_OS: [[ os_type ]] + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + +- name: resource_group_v2_[[ os_type ]] + plan: + - in_parallel: + steps: + - get: gpdb_src + passed: [gate_cli_start] + - get: gpdb_binary + resource: bin_gpdb + passed: [gate_cli_start] + trigger: true + - get: ccp_src + - get: ccp-image + - get: terraform.d + params: + unpack: true + - put: terraform + params: + <<: *ccp_default_params + vars: + <<: *ccp_default_vars + instance_type: n1-standard-2 + PLATFORM: [[ os_type ]]-cgroup2-gpdb7 + - task: gen_cluster + file: ccp_src/ci/tasks/gen_cluster.yml + params: + <<: *ccp_gen_cluster_default_params + PLATFORM: [[ os_type ]]-cgroup2-gpdb7 + - task: gpinitsystem + file: ccp_src/ci/tasks/gpinitsystem.yml + params: + <<: *default_platform + - task: run_tests + file: gpdb_src/concourse/tasks/ic_gpdb_resgroup_v2.yml + image: ccp-image + params: + TEST_OS: [[ os_type ]] + on_success: + <<: *ccp_destroy + ensure: + <<: *set_failed + - name: cli_cross_subnet plan: - in_parallel: @@ -1344,11 +1802,12 @@ jobs: cp cluster_2/.ssh/*.pem cluster_env_files/.ssh cat cluster_2/.ssh/known_hosts >> cluster_env_files/.ssh/known_hosts cat cluster_2/.ssh/config >> cluster_env_files/.ssh/config - sed -i '/^Host mdw-1/ s/$/ mdw/' cluster_env_files/.ssh/config + sed -i '/^Host cdw-1/ s/$/ cdw/' cluster_env_files/.ssh/config - task: gpinitsystem file: ccp_src/ci/tasks/gpinitsystem.yml params: - GPINIT_STANDBY_MASTER: '-s mdw-2' + GPINIT_STANDBY_COORDINATOR: '-s cdw-2' + <<: *default_platform - task: cross_subnet_tests file: gpdb_src/concourse/tasks/run_behave_on_ccp_cluster.yml image: ccp-image @@ -1488,9 +1947,7 @@ jobs: {% endif %} -{% endif %} # end CLI test_sections - -{% if pipeline_configuration == "prod" or build_test_rc_rpm or directed_release %} +{% if ( ("release" in test_sections and os_type == compile_platform ) or build_test_rc_rpm or directed_release) %} ## ====================================================================== ## ____ _ ## | _ \ ___| | ___ __ _ ___ ___ @@ -1507,60 +1964,16 @@ jobs: - get: gpdb_src trigger: true passed: - - check_format - - compile_gpdb_rhel8 - - icw_gporca_rhel8 - - icw_gporca_jit_rhel8 - - icw_planner_rhel8 - - icw_planner_jit_rhel8 - - icw_gporca_icproxy_rhel8 - - icw_planner_icproxy_rhel8 - - icw_gporca_ictcp_rhel8 - - icw_mirrorless_rhel8 - - icw_extensions_gpcloud_rhel8 - - cli_cross_subnet - - unit_tests_gporca_rhel8 - - test_gpdb_clients_rhel8 - - gpdb_pitr_rhel8 -{% for test in CLI_BEHAVE_TESTS %} -{% if "rhel8" in os_types %} - - [[ test.name ]]_rhel8 - - resource_group_rhel8 - - interconnect_rhel8 -{% endif %} -{% endfor %} - - pg_upgrade_rhel8 - - gpexpand_rhel8 - - check_centos - - get: bin_gpdb_rhel8 + - gate_icw_end + - gate_cli_end + - get: bin_gpdb + params: + skip_download: 'true' trigger: true passed: - - compile_gpdb_rhel8 - - icw_planner_rhel8 - - icw_planner_jit_rhel8 - - icw_gporca_rhel8 - - icw_gporca_jit_rhel8 - - icw_gporca_icproxy_rhel8 - - icw_planner_icproxy_rhel8 - - icw_gporca_ictcp_rhel8 - - icw_mirrorless_rhel8 - - icw_extensions_gpcloud_rhel8 - - resource_group_rhel8 - - interconnect_rhel8 - - cli_cross_subnet - {% for test in CLI_BEHAVE_TESTS %} - - [[ test.name ]]_rhel8 -{% endfor %} - - get: bin_gpdb_clients_rhel8 - passed: - - compile_gpdb_rhel8 - -{% if not directed_release %} -- name: Publish Server Builds - plan: - - in_parallel: - steps: - - get: gpdb_src + - gate_icw_end + - gate_cli_end + - get: bin_gpdb_clients trigger: true passed: - compile_gpdb_rhel8 @@ -1576,10 +1989,12 @@ jobs: steps: - put: server-build-rhel8 params: - file: output/server-build-*-rhel8*.tar.gz + skip_download: 'true' + passed: + - gate_icw_end -{% endif %} -- name: Release_Candidate +- name: release_candidate + old_name: Release_Candidate plan: - in_parallel: steps: @@ -1599,10 +2014,25 @@ jobs: steps: {% if not directed_release %} - task: rename_rc_artifacts - image: gpdb7-rhel8-build - file: gpdb_src/concourse/tasks/rename_rc_artifacts.yml - params: - RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) + image: gpdb7-[[ compile_platform ]]-build + config: + platform: linux + inputs: + - name: gpdb_src + - name: bin_gpdb + - name: bin_gpdb_clients + outputs: + - name: release_candidates + params: + RC_BUILD_TYPE_GCS: ((rc-build-type-gcs)) + run: + path: bash + args: + - -ec + - | + gpdb_semver=$(gpdb_src/getversion | cut -d' ' -f1) + cp -v bin_gpdb/bin_gpdb.tar.gz release_candidates/server-rc-${gpdb_semver}-[[ dist ]]_x86_64${RC_BUILD_TYPE_GCS}.tar.gz + cp -v bin_gpdb_clients/bin_gpdb_clients.tar.gz release_candidates/clients-rc-${gpdb_semver}-[[ dist ]]_x86_64${RC_BUILD_TYPE_GCS}.tar.gz - task: verify_gpdb_versions image: gpdb7-rhel8-build file: gpdb_src/concourse/tasks/verify_gpdb_versions.yml @@ -1620,21 +2050,23 @@ jobs: - in_parallel: steps: {% if not directed_release %} - - put: bin_gpdb_rhel8_rc - params: - file: "release_candidates/server-rc-*rhel8*.tar.gz" - - put: bin_gpdb_clients_rhel8_rc - params: - file: "release_candidates/clients-rc-*rhel8*.tar.gz" -{% endif %} + {% if os_type == default_os_type %} - put: server_src_rc params: file: "server_tar/*server-src-rc-*.tar.gz" - + {% endif %} + - put: bin_gpdb_rc + params: + file: "release_candidates/server-rc-*[[ dist ]]*.tar.gz" + - put: bin_gpdb_clients_rc + params: + file: "release_candidates/clients-rc-*[[ dist ]]*.tar.gz" +{% endif %} {% endif %} -{% if not directed_release %} -- name: Build_Release_Candidate_RPMs +{% if not directed_release and os_type == compile_platform %} +- name: build_release_candidate_rpms + old_name: Build_Release_Candidate_RPMs plan: - in_parallel: steps: @@ -1650,14 +2082,12 @@ jobs: {% if not build_test_rc_rpm %} passed: [gate_release_candidate_start] {% endif %} - - get: gpdb7-rhel8-build + - get: gpdb7-[[ compile_platform ]]-build - get: greenplum-database-release - get: license_file - task: create_gpdb_rpm_package_rhel8 file: greenplum-database-release/ci/concourse/tasks/build-gpdb-rpm.yml - image: gpdb7-rhel8-build - input_mapping: - bin_gpdb: bin_gpdb_rhel8 + image: gpdb7-[[ compile_platform ]]-build output_mapping: gpdb_rpm_installer: gpdb_rpm_rhel8 params: @@ -1668,7 +2098,7 @@ jobs: GPDB_URL: https://github.com/greenplum-db/gpdb GPDB_OSS: false - task: rename_rc_artifacts - image: gpdb7-rhel8-build + image: gpdb7-[[ compile_platform ]]-build config: platform: linux inputs: @@ -1687,9 +2117,9 @@ jobs: {% if build_test_rc_rpm %} # Uniquely name the rpm using the github username and test branch # to prevent filename collisions on GCS. - cp -v gpdb_rpm_rhel8/*.rpm renamed_gpdb_rpm_rhel8/greenplum-db-[[ git_username ]]-[[ git_branch ]]-${gpdb_semver}-rhel8-x86_64${RC_BUILD_TYPE_GCS}.rpm + cp -v gpdb_rpm_[[ os_type ]]/*.rpm renamed_gpdb_rpm_[[ os_type ]]/greenplum-db-[[ git_username ]]-[[ git_branch ]]-${gpdb_semver}-[[ dist ]]-x86_64${RC_BUILD_TYPE_GCS}.rpm {% else %} - cp -v gpdb_rpm_rhel8/*.rpm renamed_gpdb_rpm_rhel8/greenplum-db-${gpdb_semver}-rhel8-x86_64${RC_BUILD_TYPE_GCS}.rpm + cp -v gpdb_rpm_[[ os_type ]]/*.rpm renamed_gpdb_rpm_[[ os_type ]]/greenplum-db-${gpdb_semver}-[[ dist ]]-x86_64${RC_BUILD_TYPE_GCS}.rpm {% endif %} - in_parallel: steps: diff --git a/concourse/scripts/backup_utils.sh b/concourse/scripts/backup_utils.sh index 21314e355d0e..a708797c2f56 100644 --- a/concourse/scripts/backup_utils.sh +++ b/concourse/scripts/backup_utils.sh @@ -4,7 +4,7 @@ destroy_gpdb() { # Setup environment source /usr/local/greenplum-db-devel/greenplum_path.sh; export PGPORT=5432; - export COORDINATOR_DATA_DIRECTORY=/data/gpdata/master/gpseg-1; + export COORDINATOR_DATA_DIRECTORY=/data/gpdata/coordinator/gpseg-1; yes|gpdeletesystem -f -d $COORDINATOR_DATA_DIRECTORY rm -rf $GPHOME/* diff --git a/concourse/scripts/builds/GpBuild.py b/concourse/scripts/builds/GpBuild.py index 5139cb412605..f8369f8e5d46 100755 --- a/concourse/scripts/builds/GpBuild.py +++ b/concourse/scripts/builds/GpBuild.py @@ -52,7 +52,7 @@ def _run_gpdb_command(self, command, stdout=None, stderr=None, source_env_cmd='' cmd = source_env_cmd runcmd = "runuser gpadmin -c \"{0} && {1} \"".format(cmd, command) if print_command: - print "Executing {}".format(runcmd) + print("Executing {}".format(runcmd)) return subprocess.call([runcmd], shell=True, stdout=stdout, stderr=stderr) def run_explain_test_suite(self, dbexists): @@ -74,7 +74,7 @@ def run_explain_test_suite(self, dbexists): status = self._run_gpdb_command("psql -q -f stats.sql", stdout=f) if status: with open("load_stats.txt", "r") as f: - print f.read() + print(f.read()) fail_on_error(status) # set gucs if any were specified @@ -95,10 +95,10 @@ def run_explain_test_suite(self, dbexists): if fsql.endswith('.sql') and fsql not in ['stats.sql', 'schema.sql']: output_fname = 'out/{}'.format(fsql.replace('.sql', '.out')) with open(output_fname, 'w') as fout: - print "Running query: " + fsql + print("Running query: " + fsql) current_status = self._run_gpdb_command("psql -a -f sql/{}".format(fsql), stdout=fout, stderr=fout, source_env_cmd=source_env_cmd, print_command=False) if current_status != 0: - print "ERROR: {0}".format(current_status) + print("ERROR: {0}".format(current_status)) status = status if status != 0 else current_status return status diff --git a/concourse/scripts/compile_gpdb.bash b/concourse/scripts/compile_gpdb.bash index d0bfd1fb3b25..0e2c8da8b3b7 100755 --- a/concourse/scripts/compile_gpdb.bash +++ b/concourse/scripts/compile_gpdb.bash @@ -151,7 +151,7 @@ function _main() { unittest_check_gpdb fi - include_dependencies +# include_dependencies export_gpdb export_gpdb_extensions diff --git a/concourse/scripts/compile_gpdb_remote_windows.bash b/concourse/scripts/compile_gpdb_remote_windows.bash index 3cb7807f0502..ba76af9c4e99 100755 --- a/concourse/scripts/compile_gpdb_remote_windows.bash +++ b/concourse/scripts/compile_gpdb_remote_windows.bash @@ -66,6 +66,9 @@ EOF } function download() { + # On RHEL9 platform, the path C:\Users\buildbot cannot be escaped for scp + # So replace all backslash with double backslashes + WORK_DIR=${WORK_DIR//\\/\\\\} pushd "$ROOT_DIR/gpdb_artifacts/" scp -P "${REMOTE_PORT}" -q "${REMOTE_USER}"@"${REMOTE_HOST}":"${WORK_DIR}/*.msi" ./ scp -P "${REMOTE_PORT}" -q "${REMOTE_USER}"@"${REMOTE_HOST}":"${WORK_DIR}/*.exe" ./ diff --git a/concourse/scripts/configurations/pg_upgrade_gpinitsystem_config b/concourse/scripts/configurations/pg_upgrade_gpinitsystem_config index 4a01e21ef8d2..6de292b1d8ca 100644 --- a/concourse/scripts/configurations/pg_upgrade_gpinitsystem_config +++ b/concourse/scripts/configurations/pg_upgrade_gpinitsystem_config @@ -22,11 +22,11 @@ PORT_BASE=20000 declare -a DATA_DIRECTORY=(/data/gpdata/primary /data/gpdata/primary) #### OS-configured hostname or IP address of the master host. -MASTER_HOSTNAME=mdw +MASTER_HOSTNAME=cdw #### File system location where the master data directory #### will be created. -MASTER_DIRECTORY=/data/gpdata/master +MASTER_DIRECTORY=/data/gpdata/coordinator #### Port number for the master instance. MASTER_PORT=5432 diff --git a/concourse/scripts/ic_gpdb_resgroup.bash b/concourse/scripts/ic_gpdb_resgroup.bash index d4671783c926..f8fd29c27e65 100755 --- a/concourse/scripts/ic_gpdb_resgroup.bash +++ b/concourse/scripts/ic_gpdb_resgroup.bash @@ -58,7 +58,7 @@ run_resgroup_test() { ssh $gpdb_master_alias bash -ex < ~/gpdb-env.sh <<'EOF' source /usr/local/greenplum-db-devel/greenplum_path.sh export PGPORT=5432 - export COORDINATOR_DATA_DIRECTORY=/data/gpdata/master/gpseg-1 + export COORDINATOR_DATA_DIRECTORY=/data/gpdata/coordinator/gpseg-1 export PGDATABASE=gptest alias cdd='cd \$COORDINATOR_DATA_DIRECTORY' diff --git a/concourse/scripts/test_upgrade.bash b/concourse/scripts/test_upgrade.bash index 75ffe5cec45e..7a38336e8597 100755 --- a/concourse/scripts/test_upgrade.bash +++ b/concourse/scripts/test_upgrade.bash @@ -371,12 +371,12 @@ done # Set up the globals according to whether we're running in local or Concourse # mode. if (( $CONCOURSE_MODE )); then - MASTER_HOST=mdw + MASTER_HOST=cdw OLD_GPHOME=/usr/local/greenplum-db-devel NEW_GPHOME=/usr/local/gpdb_master DATADIR_PREFIX=/data/gpdata - OLD_COORDINATOR_DATA_DIRECTORY=/data/gpdata/master/gpseg-1 - NEW_COORDINATOR_DATA_DIRECTORY=/data/gpdata/master-new/gpseg-1 + OLD_COORDINATOR_DATA_DIRECTORY=/data/gpdata/coordinator/gpseg-1 + NEW_COORDINATOR_DATA_DIRECTORY=/data/gpdata/coordinator-new/gpseg-1 PSQL_ADDOPTS= GPINITSYSTEM_CONFIG=gpinitsystem_config GPINITSYSTEM_HOSTFILE=segment_host_list diff --git a/concourse/scripts/unit_tests_gporca.bash b/concourse/scripts/unit_tests_gporca.bash index 458185e5be0c..a8dd0376fff6 100755 --- a/concourse/scripts/unit_tests_gporca.bash +++ b/concourse/scripts/unit_tests_gporca.bash @@ -8,7 +8,7 @@ function build_xerces OUTPUT_DIR="gpdb_src/gpAux/ext/${BLD_ARCH}" mkdir -p xerces_patch/concourse cp -r gpdb_src/src/backend/gporca/concourse/xerces-c xerces_patch/concourse - /usr/bin/python xerces_patch/concourse/xerces-c/build_xerces.py --output_dir=${OUTPUT_DIR} + /usr/bin/python3 xerces_patch/concourse/xerces-c/build_xerces.py --output_dir=${OUTPUT_DIR} rm -rf build } @@ -27,7 +27,7 @@ function test_orca function _main { mkdir gpdb_src/gpAux/ext - build_xerces + #build_xerces test_orca } diff --git a/concourse/tasks/run_behave_on_ccp_cluster.yml b/concourse/tasks/run_behave_on_ccp_cluster.yml index 8cb362b224f3..8e1e6fb50554 100644 --- a/concourse/tasks/run_behave_on_ccp_cluster.yml +++ b/concourse/tasks/run_behave_on_ccp_cluster.yml @@ -17,7 +17,7 @@ run: ccp_src/scripts/setup_ssh_to_cluster.sh # TODO: ask CCP maintainers for a feature to do this for us - scp cluster_env_files/hostfile_all mdw:/tmp + scp cluster_env_files/hostfile_all cdw:/tmp # Install patchelf. We need to SSH as root, hence the use of # cluster_env_files. @@ -27,17 +27,17 @@ run: # Set max_connections values if specified if (( $COORDINATOR_MAX_CONNECT > 0 && $SEGMENT_MAX_CONNECT > 0 )); then - ssh -n mdw " + ssh -n cdw " source /usr/local/greenplum-db-devel/greenplum_path.sh - export COORDINATOR_DATA_DIRECTORY='/data/gpdata/master/gpseg-1' + export COORDINATOR_DATA_DIRECTORY='/data/gpdata/coordinator/gpseg-1' gpconfig -c max_connections -v $SEGMENT_MAX_CONNECT -m $COORDINATOR_MAX_CONNECT " fi # ensure cluster is stopped - ssh -n mdw " + ssh -n cdw " source /usr/local/greenplum-db-devel/greenplum_path.sh - export COORDINATOR_DATA_DIRECTORY='/data/gpdata/master/gpseg-1' + export COORDINATOR_DATA_DIRECTORY='/data/gpdata/coordinator/gpseg-1' if [ -d \$COORDINATOR_DATA_DIRECTORY ]; then gpstop -a; fi " @@ -84,20 +84,20 @@ run: # One use is to modify networking on the cluster machines, as is done to # add/delete blackhole routes in the gprecoverseg_newhost tests. SUDO_ACCESS_ON_HOSTS=${SUDO_ACCESS_ON_HOSTS:-0} - SUDO_MDW_USERNAME=${SUDO_MDW_USERNAME:-centos} + SUDO_CDW_USERNAME=${SUDO_CDW_USERNAME:-centos} if (( $SUDO_ACCESS_ON_HOSTS )); then while read LINE; do host=$(echo $LINE | awk '{ print $3 }') - ssh -n $SUDO_MDW_USERNAME@$host " + ssh -n $SUDO_CDW_USERNAME@$host " sudo usermod -a -G google-sudoers gpadmin " done ~/.ssh/id_rsa.pub; for i in mdw sdw{1..4}; do ssh $SUDO_MDW_USERNAME@\$i \"sudo chmod 777 /usr/local\"; done'" - ssh -t mdw 'mkdir -p /home/gpadmin/sqldump' - scp sqldump/* mdw:/home/gpadmin/sqldump/ - ssh -t mdw 'xz -d /home/gpadmin/sqldump/dump.sql.xz' + SUDO_CDW_USERNAME=${SUDO_CDW_USERNAME:-rhel} + ssh -t cdw "HOST1=cdw HOST2=sdw1 HOST3=sdw2 HOST4=sdw3 HOST5=sdw4 \ + bash -c 'ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub; for i in cdw sdw{1..4}; do ssh $SUDO_CDW_USERNAME@\$i \"sudo chmod 777 /usr/local\"; done'" + ssh -t cdw 'mkdir -p /home/gpadmin/sqldump' + scp sqldump/* cdw:/home/gpadmin/sqldump/ + ssh -t cdw 'xz -d /home/gpadmin/sqldump/dump.sql.xz' diff --git a/contrib/extprotocol/expected/exttableext.out b/contrib/extprotocol/expected/exttableext.out index 7c68cb2c80ec..04faefe2c11d 100644 --- a/contrib/extprotocol/expected/exttableext.out +++ b/contrib/extprotocol/expected/exttableext.out @@ -1559,6 +1559,14 @@ ERROR: protocol attribute "validatorproc" not recognized drop role if exists demoprot_nopriv; create role demoprot_nopriv with login ; NOTICE: resource queue required -- using default resource queue "pg_default" +-- start_matchsubs +-- m/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ +-- s/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ERROR: internal error: demoprot called with a different protocol \(demoprot_new\)/ +-- end_matchsubs +-- start_matchsubs +-- m/line \d+ of file/ +-- s/line \d+ of file/file/ +-- end_matchsubs -- Test 92: Rename existing protocol DROP FUNCTION IF EXISTS url_validator(); CREATE OR REPLACE FUNCTION url_validator() RETURNS void AS @@ -1630,7 +1638,7 @@ ERROR: protocol "demoprot" does not exist (seg0 slice1 rh55-qavm55:7532 pid=18 -- However demoprot implementation prevents using any other protocol name than "demoprot" -- therefore the error is expected. select count(*) from exttabtest_r_new; -ERROR: internal error: demoprot called with a different protocol (demoprot_new) (gpextprotocol.c:108) (seg0 slice1 rh55-qavm55:7532 pid=18394) (cdbdisp.c:1453) +ERROR: internal error: demoprot called with a different protocol (demoprot_new) CONTEXT: External table exttabtest_r_new, file demoprot_new://exttabtest.txt -- Rename protocol name back to demoprot ALTER PROTOCOL demoprot_new RENAME to demoprot; diff --git a/contrib/extprotocol/sql/exttableext.sql b/contrib/extprotocol/sql/exttableext.sql index 227475263f85..081c4665c341 100644 --- a/contrib/extprotocol/sql/exttableext.sql +++ b/contrib/extprotocol/sql/exttableext.sql @@ -1252,6 +1252,14 @@ validatorfunc = url_validator drop role if exists demoprot_nopriv; create role demoprot_nopriv with login ; +-- start_matchsubs +-- m/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ +-- s/^ERROR: internal error: demoprot called with a different protocol \(demoprot_new\).*/ERROR: internal error: demoprot called with a different protocol \(demoprot_new\)/ +-- end_matchsubs +-- start_matchsubs +-- m/line \d+ of file/ +-- s/line \d+ of file/file/ +-- end_matchsubs -- Test 92: Rename existing protocol DROP FUNCTION IF EXISTS url_validator(); diff --git a/contrib/pgcrypto/Makefile b/contrib/pgcrypto/Makefile index 36a8403a9fb8..cc802e9ea833 100644 --- a/contrib/pgcrypto/Makefile +++ b/contrib/pgcrypto/Makefile @@ -30,14 +30,13 @@ DATA = pgcrypto--1.3.sql pgcrypto--1.2--1.3.sql pgcrypto--1.1--1.2.sql \ pgcrypto--1.0--1.1.sql pgcrypto--unpackaged--1.0.sql PGFILEDESC = "pgcrypto - cryptographic functions" -REGRESS_OPTS = --dbname=$(CONTRIB_TESTDB) --init-file=$(top_builddir)/src/test/regress/init_file +REGRESS_OPTS = --dbname=$(CONTRIB_TESTDB) --init-file=$(top_builddir)/src/test/regress/init_file REGRESS = init md5 sha1 hmac-md5 hmac-sha1 blowfish rijndael \ $(CF_TESTS) \ crypt-des crypt-md5 crypt-blowfish crypt-xdes \ pgp-armor pgp-decrypt pgp-encrypt $(CF_PGP_TESTS) \ - pgp-pubkey-decrypt pgp-pubkey-encrypt pgp-info \ - setup_fips fips + pgp-pubkey-decrypt pgp-pubkey-encrypt pgp-info EXTRA_CLEAN = gen-rtab diff --git a/contrib/pgcrypto/expected/blowfish_1.out b/contrib/pgcrypto/expected/blowfish_1.out new file mode 100644 index 000000000000..565a0853cb03 --- /dev/null +++ b/contrib/pgcrypto/expected/blowfish_1.out @@ -0,0 +1,95 @@ +-- +-- Blowfish cipher +-- +-- ensure consistent test output regardless of the default bytea format +SET bytea_output TO escape; +-- some standard Blowfish testvalues +SELECT encode(encrypt( +decode('0000000000000000', 'hex'), +decode('0000000000000000', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('ffffffffffffffff', 'hex'), +decode('ffffffffffffffff', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('1000000000000001', 'hex'), +decode('3000000000000000', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('1111111111111111', 'hex'), +decode('1111111111111111', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('0123456789abcdef', 'hex'), +decode('fedcba9876543210', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('01a1d6d039776742', 'hex'), +decode('fedcba9876543210', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +SELECT encode(encrypt( +decode('ffffffffffffffff', 'hex'), +decode('0000000000000000', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- setkey +SELECT encode(encrypt( +decode('fedcba9876543210', 'hex'), +decode('f0e1d2c3b4a5968778695a4b3c2d1e0f', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- with padding +SELECT encode(encrypt( +decode('01234567890123456789', 'hex'), +decode('33443344334433443344334433443344', 'hex'), +'bf-ecb'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- cbc +-- 28 bytes key +SELECT encode(encrypt( +decode('6b77b4d63006dee605b156e27403979358deb9e7154616d959f1652bd5', 'hex'), +decode('37363534333231204e6f77206973207468652074696d6520666f7220', 'hex'), +'bf-cbc'), 'hex'); +ERROR: encrypt error: Key was too big +-- 29 bytes key +SELECT encode(encrypt( +decode('6b77b4d63006dee605b156e27403979358deb9e7154616d959f1652bd5ff92cc', 'hex'), +decode('37363534333231204e6f77206973207468652074696d6520666f722000', 'hex'), +'bf-cbc'), 'hex'); +ERROR: encrypt error: Key was too big +-- blowfish-448 +SELECT encode(encrypt( +decode('fedcba9876543210', 'hex'), +decode('f0e1d2c3b4a5968778695a4b3c2d1e0f001122334455667704689104c2fd3b2f584023641aba61761f1f1f1f0e0e0e0effffffffffffffff', 'hex'), +'bf-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Key was too big +-- result: c04504012e4e1f53 +-- empty data +select encode(encrypt('', 'foo', 'bf'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 10 bytes key +select encode(encrypt('foo', '0123456789', 'bf'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 22 bytes key +select encode(encrypt('foo', '0123456789012345678901', 'bf'), 'hex'); +ERROR: encrypt error: Key was too big +-- decrypt +select decrypt(encrypt('foo', '0123456', 'bf'), '0123456', 'bf'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- iv +select encode(encrypt_iv('foo', '0123456', 'abcd', 'bf'), 'hex'); +ERROR: encrypt_iv error: Cipher cannot be initialized ? +select decrypt_iv(decode('95c7e89322525d59', 'hex'), '0123456', 'abcd', 'bf'); +ERROR: decrypt_iv error: Cipher cannot be initialized ? +-- long message +select encode(encrypt('Lets try a longer message.', '0123456789', 'bf'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +select decrypt(encrypt('Lets try a longer message.', '0123456789', 'bf'), '0123456789', 'bf'); +ERROR: encrypt error: Cipher cannot be initialized ? diff --git a/contrib/pgcrypto/expected/cast5_1.out b/contrib/pgcrypto/expected/cast5_1.out new file mode 100644 index 000000000000..e3b38dbce2fe --- /dev/null +++ b/contrib/pgcrypto/expected/cast5_1.out @@ -0,0 +1,48 @@ +-- +-- Cast5 cipher +-- +-- ensure consistent test output regardless of the default bytea format +SET bytea_output TO escape; +-- test vectors from RFC2144 +-- 128 bit key +SELECT encode(encrypt( +decode('01 23 45 67 89 AB CD EF', 'hex'), +decode('01 23 45 67 12 34 56 78 23 45 67 89 34 56 78 9A', 'hex'), +'cast5-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- result: 23 8B 4F E5 84 7E 44 B2 +-- 80 bit key +SELECT encode(encrypt( +decode('01 23 45 67 89 AB CD EF', 'hex'), +decode('01 23 45 67 12 34 56 78 23 45', 'hex'), +'cast5-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- result: EB 6A 71 1A 2C 02 27 1B +-- 40 bit key +SELECT encode(encrypt( +decode('01 23 45 67 89 AB CD EF', 'hex'), +decode('01 23 45 67 12', 'hex'), +'cast5-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- result: 7A C8 16 D1 6E 9B 30 2E +-- cbc +-- empty data +select encode( encrypt('', 'foo', 'cast5'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 10 bytes key +select encode( encrypt('foo', '0123456789', 'cast5'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- decrypt +select decrypt(encrypt('foo', '0123456', 'cast5'), '0123456', 'cast5'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- iv +select encode(encrypt_iv('foo', '0123456', 'abcd', 'cast5'), 'hex'); +ERROR: encrypt_iv error: Cipher cannot be initialized ? +select decrypt_iv(decode('384a970695ce016a', 'hex'), + '0123456', 'abcd', 'cast5'); +ERROR: decrypt_iv error: Cipher cannot be initialized ? +-- long message +select encode(encrypt('Lets try a longer message.', '0123456789', 'cast5'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +select decrypt(encrypt('Lets try a longer message.', '0123456789', 'cast5'), '0123456789', 'cast5'); +ERROR: encrypt error: Cipher cannot be initialized ? diff --git a/contrib/pgcrypto/expected/des_1.out b/contrib/pgcrypto/expected/des_1.out new file mode 100644 index 000000000000..e8cca0505fe1 --- /dev/null +++ b/contrib/pgcrypto/expected/des_1.out @@ -0,0 +1,31 @@ +-- +-- DES cipher +-- +-- ensure consistent test output regardless of the default bytea format +SET bytea_output TO escape; +-- no official test vectors atm +-- from blowfish.sql +SELECT encode(encrypt( +decode('0123456789abcdef', 'hex'), +decode('fedcba9876543210', 'hex'), +'des-ecb/pad:none'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- empty data +select encode( encrypt('', 'foo', 'des'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- 8 bytes key +select encode( encrypt('foo', '01234589', 'des'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- decrypt +select decrypt(encrypt('foo', '0123456', 'des'), '0123456', 'des'); +ERROR: encrypt error: Cipher cannot be initialized ? +-- iv +select encode(encrypt_iv('foo', '0123456', 'abcd', 'des'), 'hex'); +ERROR: encrypt_iv error: Cipher cannot be initialized ? +select decrypt_iv(decode('50735067b073bb93', 'hex'), '0123456', 'abcd', 'des'); +ERROR: decrypt_iv error: Cipher cannot be initialized ? +-- long message +select encode(encrypt('Lets try a longer message.', '01234567', 'des'), 'hex'); +ERROR: encrypt error: Cipher cannot be initialized ? +select decrypt(encrypt('Lets try a longer message.', '01234567', 'des'), '01234567', 'des'); +ERROR: encrypt error: Cipher cannot be initialized ? diff --git a/contrib/pgcrypto/expected/fips_1.out b/contrib/pgcrypto/expected/fips_1.out deleted file mode 100644 index 04021993b337..000000000000 --- a/contrib/pgcrypto/expected/fips_1.out +++ /dev/null @@ -1,131 +0,0 @@ -ALTER DATABASE contrib_regression SET pgcrypto.fips TO on; -ERROR: FIPS enabled OpenSSL is required for strict FIPS mode (openssl.c:1055) -HINT: Recompile OpenSSL with the FIPS module, or install a FIPS enabled OpenSSL distribution. -\c contrib_regression -SHOW pgcrypto.fips; - pgcrypto.fips ---------------- - off -(1 row) - -CREATE TABLE fipstest (data text, res text, salt text); -NOTICE: Table doesn't have 'DISTRIBUTED BY' clause -- Using column named 'data' as the Greenplum Database data distribution key for this table. -HINT: The 'DISTRIBUTED BY' clause determines the distribution of data. Make sure column(s) chosen are the optimal data distribution key to minimize skew. -INSERT INTO fipstest VALUES ('password', '', ''); -SELECT 'Test digest md5: EXPECTED ERROR FAIL FIPS' as comment; - comment -------------------------------------------- - Test digest md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT digest('santa claus', 'md5'); - digest ------------------------------------- - \x7c4f44266cee2dbfacaa009ef5e167dc -(1 row) - -SELECT 'Test digest sha256: EXPECTED PASS' as comment; - comment ------------------------------------ - Test digest sha256: EXPECTED PASS -(1 row) - -SELECT digest('santa claus', 'sha256'); - digest --------------------------------------------------------------------- - \x675b8f61fc27140b5f06fec4613d8b3d9b913a82074d4c790374558c18d2cb7d -(1 row) - -SELECT 'Test hmac md5: EXPECTED ERROR FAIL FIPS' as comment; - comment ------------------------------------------ - Test hmac md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'md5'); - hmac ------------------------------------- - \xd13b4c8c8a6e9d6236e8cc0b141968c5 -(1 row) - -SELECT 'Test hmac sha256: EXPECTED PASS' as comment; - comment ---------------------------------- - Test hmac sha256: EXPECTED PASS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'sha256'); - hmac --------------------------------------------------------------------- - \xc88fe8ff0541b1bb25abd971fa7642d256a1c0109f7e56875d593a3daaeacf54 -(1 row) - -SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; - comment ------------------------------------- - Test gen_salt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET salt = gen_salt('md5'); -SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------- - Test crypt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET res = crypt(data, salt); -SELECT res = crypt(data, res) AS "worked" FROM fipstest; - worked --------- - t -(1 row) - -SELECT 'Test pgp : EXPECTED PASS' as comment; - comment --------------------------- - Test pgp : EXPECTED PASS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); - pgp_sym_decrypt ------------------ - santa clause -(1 row) - -SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; - comment -------------------------------- - Test pgp : EXPECTED FAIL FIPS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=bf'), 'mypass'); - pgp_sym_decrypt ------------------ - santa clause -(1 row) - -SELECT 'Test raw encrypt : EXPECTED PASS' as comment; - comment ----------------------------------- - Test raw encrypt : EXPECTED PASS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; - raw_aes ------------------------------------- - \xe7ac3ad57bde59c5b78c08805afdb774 -(1 row) - -SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------------- - Test raw encrypt : EXPECTED FAIL FIPS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; - raw_blowfish ------------------------------------- - \x0f4fe496594a2e762fb29a22fc6750e2 -(1 row) - -DROP TABLE fipstest; diff --git a/contrib/pgcrypto/expected/fips_2.out b/contrib/pgcrypto/expected/fips_2.out index 51957b898daa..e69de29bb2d1 100644 --- a/contrib/pgcrypto/expected/fips_2.out +++ b/contrib/pgcrypto/expected/fips_2.out @@ -1,101 +0,0 @@ -ALTER DATABASE contrib_regression SET pgcrypto.fips TO on; -\c contrib_regression -SHOW pgcrypto.fips; - pgcrypto.fips ---------------- - on -(1 row) - -CREATE TABLE fipstest (data text, res text, salt text); -INSERT INTO fipstest VALUES ('password', '', ''); -SELECT 'Test digest md5: EXPECTED ERROR FAIL FIPS' as comment; - comment -------------------------------------------- - Test digest md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT digest('santa claus', 'md5'); -ERROR: Cannot use "md5": -SELECT 'Test digest sha256: EXPECTED PASS' as comment; - comment ------------------------------------ - Test digest sha256: EXPECTED PASS -(1 row) - -SELECT digest('santa claus', 'sha256'); - digest --------------------------------------------------------------------- - \x675b8f61fc27140b5f06fec4613d8b3d9b913a82074d4c790374558c18d2cb7d -(1 row) - -SELECT 'Test hmac md5: EXPECTED ERROR FAIL FIPS' as comment; - comment ------------------------------------------ - Test hmac md5: EXPECTED ERROR FAIL FIPS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'md5'); -ERROR: Cannot use "md5": -SELECT 'Test hmac sha256: EXPECTED PASS' as comment; - comment ---------------------------------- - Test hmac sha256: EXPECTED PASS -(1 row) - -SELECT hmac('santa claus', 'aaa', 'sha256'); - hmac --------------------------------------------------------------------- - \xc88fe8ff0541b1bb25abd971fa7642d256a1c0109f7e56875d593a3daaeacf54 -(1 row) - -SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; - comment ------------------------------------- - Test gen_salt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET salt = gen_salt('md5'); -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:213) -SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------- - Test crypt : EXPECTED FAIL FIPS -(1 row) - -UPDATE fipstest SET res = crypt(data, salt); -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:266) -SELECT res = crypt(data, res) AS "worked" FROM fipstest; -ERROR: requested functionality not allowed in FIPS mode (pgcrypto.c:266) -SELECT 'Test pgp : EXPECTED PASS' as comment; - comment --------------------------- - Test pgp : EXPECTED PASS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) -SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; - comment -------------------------------- - Test pgp : EXPECTED FAIL FIPS -(1 row) - -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=bf'), 'mypass'); -ERROR: Unsupported cipher algorithm -SELECT 'Test raw encrypt : EXPECTED PASS' as comment; - comment ----------------------------------- - Test raw encrypt : EXPECTED PASS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) -SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; - comment ---------------------------------------- - Test raw encrypt : EXPECTED FAIL FIPS -(1 row) - -SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; -ERROR: requested functionality not allowed in FIPS mode (openssl.c:772) -DROP TABLE fipstest; diff --git a/contrib/pgcrypto/expected/pgp-decrypt_1.out b/contrib/pgcrypto/expected/pgp-decrypt_1.out new file mode 100644 index 000000000000..63d5ab98654f --- /dev/null +++ b/contrib/pgcrypto/expected/pgp-decrypt_1.out @@ -0,0 +1,421 @@ +-- +-- pgp decrypt tests +-- +-- Checking ciphers +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.blowfish.sha1.mdc.s2k3.z0 + +jA0EBAMCfFNwxnvodX9g0jwB4n4s26/g5VmKzVab1bX1SmwY7gvgvlWdF3jKisvS +yA6Ce1QTMK3KdL2MPfamsTUSAML8huCJMwYQFfE= +=JcP+ +-----END PGP MESSAGE----- +'), 'foobar'); +ERROR: Wrong key or corrupt data +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCci97v0Q6Z0Zg0kQBsVf5Oe3iC+FBzUmuMV9KxmAyOMyjCc/5i8f1Eest +UTAsG35A1vYs02VARKzGz6xI2UHwFUirP+brPBg3Ee7muOx8pA== +=XtrP +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k3.z0 + +jA0ECAMCI7YQpWqp3D1g0kQBCjB7GlX7+SQeXNleXeXQ78ZAPNliquGDq9u378zI +5FPTqAhIB2/2fjY8QEIs1ai00qphjX2NitxV/3Wn+6dufB4Q4g== +=rCZt +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k3.z0 + +jA0ECQMC4f/5djqCC1Rg0kQBTHEPsD+Sw7biBsM2er3vKyGPAQkuTBGKC5ie7hT/ +lceMfQdbAg6oTFyJpk/wH18GzRDphCofg0X8uLgkAKMrpcmgog== +=fB6S +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking MDC modes +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.nomdc.s2k3.z0 + +jA0EBwMCnv07rlXqWctgyS2Dm2JfOKCRL4sLSLJUC8RS2cH7cIhKSuLitOtyquB+ +u9YkgfJfsuRJmgQ9tmo= +=60ui +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCEeP3idNjQ1Bg0kQBf4G0wX+2QNzLh2YNwYkQgQkfYhn/hLXjV4nK9nsE +8Ex1Dsdt5UPvOz8W8VKQRS6loOfOe+yyXil8W3IYFwUpdDUi+Q== +=moGf +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking hashes +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.md5.mdc.s2k3.z0 + +jA0EBwMClrXXtOXetohg0kQBn0Kl1ymevQZRHkdoYRHgzCwSQEiss7zYff2UNzgO +KyRrHf7zEBuZiZ2AG34jNVMOLToj1jJUg5zTSdecUzQVCykWTA== +=NyLk +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCApbdlrURoWJg0kQBzHM/E0o7djY82bNuspjxjAcPFrrtp0uvDdMQ4z2m +/PM8jhgI5vxFYfNQjLl8y3fHYIomk9YflN9K/Q13iq8A8sjeTw== +=FxbQ +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking S2K modes +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k0.z0 + +jAQEBwAC0kQBKTaLAKE3xzps+QIZowqRNb2eAdzBw2LxEW2YD5PgNlbhJdGg+dvw +Ah9GXjGS1TVALzTImJbz1uHUZRfhJlFbc5yGQw== +=YvkV +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k1.z0 + +jAwEBwEC/QTByBLI3b/SRAHPxKzI6SZBo5lAEOD+EsvKQWO4adL9tDY+++Iqy1xK +4IaWXVKEj9R2Lr2xntWWMGZtcKtjD2lFFRXXd9dZp1ZThNDz +=dbXm +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCEq4Su3ZqNEJg0kQB4QG5jBTKF0i04xtH+avzmLhstBNRxvV3nsmB3cwl +z+9ZaA/XdSx5ZiFnMym8P6r8uY9rLjjNptvvRHlxIReF+p9MNg== +=VJKg +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k0.z0 + +jAQECAAC0kQBBDnQWkgsx9YFaqDfWmpsiyAJ6y2xG/sBvap1dySYEMuZ+wJTXQ9E +Cr3i2M7TgVZ0M4jp4QL0adG1lpN5iK7aQeOwMw== +=cg+i +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k1.z0 + +jAwECAECruOfyNDFiTnSRAEVoGXm4A9UZKkWljdzjEO/iaE7mIraltIpQMkiqCh9 +7h8uZ2u9uRBOv222fZodGvc6bvq/4R4hAa/6qSHtm8mdmvGt +=aHmC +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes192.sha1.mdc.s2k3.z0 + +jA0ECAMCjFn6SRi3SONg0kQBqtSHPaD0m7rXfDAhCWU/ypAsI93GuHGRyM99cvMv +q6eF6859ZVnli3BFSDSk3a4e/pXhglxmDYCfjAXkozKNYLo6yw== +=K0LS +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k0.z0 + +jAQECQAC0kQB4L1eMbani07XF2ZYiXNK9LW3v8w41oUPl7dStmrJPQFwsdxmrDHu +rQr3WbdKdY9ufjOE5+mXI+EFkSPrF9rL9NCq6w== +=RGts +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k1.z0 + +jAwECQECKHhrou7ZOIXSRAHWIVP+xjVQcjAVBTt+qh9SNzYe248xFTwozkwev3mO ++KVJW0qhk0An+Y2KF99/bYFl9cL5D3Tl43fC8fXGl3x3m7pR +=SUrU +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes256.sha1.mdc.s2k3.z0 + +jA0ECQMCjc8lwZu8Fz1g0kQBkEzjImi21liep5jj+3dAJ2aZFfUkohi8b3n9z+7+ +4+NRzL7cMW2RLAFnJbiqXDlRHMwleeuLN1up2WIxsxtYYuaBjA== +=XZrG +-----END PGP MESSAGE----- +'), 'foobar'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking longer passwords +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCx6dBiuqrYNRg0kQBEo63AvA1SCslxP7ayanLf1H0/hlk2nONVhTwVEWi +tTGup1mMz6Cfh1uDRErUuXpx9A0gdMu7zX0o5XjrL7WGDAZdSw== +=XKKG +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCBDvYuS990iFg0kQBW31UK5OiCjWf5x6KJ8qNNT2HZWQCjCBZMU0XsOC6 +CMxFKadf144H/vpoV9GA0f22keQgCl0EsTE4V4lweVOPTKCMJg== +=gWDh +-----END PGP MESSAGE----- +'), '0123456789abcdefghij2jk4h5g2j54khg23h54g2kh54g2khj54g23hj54'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCqXbFafC+ofVg0kQBejyiPqH0QMERVGfmPOjtAxvyG5KDIJPYojTgVSDt +FwsDabdQUz5O7bgNSnxfmyw1OifGF+W2bIn/8W+0rDf8u3+O+Q== +=OxOF +-----END PGP MESSAGE----- +'), 'x'); + pgp_sym_decrypt +----------------- + Secret message. +(1 row) + +-- Checking various data +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat1.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCGJ+SpuOysINg0kQBJfSjzsW0x4OVcAyr17O7FBvMTwIGeGcJd99oTQU8 +Xtx3kDqnhUq9Z1fS3qPbi5iNP2A9NxOBxPWz2JzxhydANlgbxg== +=W/ik +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'), 'sha1'), 'hex'); + encode +------------------------------------------ + 0225e3ede6f2587b076d021a189ff60aad67e066 +(1 row) + +-- expected: 0225e3ede6f2587b076d021a189ff60aad67e066 +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat2.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCvdpDvidNzMxg0jUBvj8eS2+1t/9/zgemxvhtc0fvdKGGbjH7dleaTJRB +SaV9L04ky1qECNDx3XjnoKLC+H7IOQ== +=Fxen +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'), 'sha1'), 'hex'); + encode +------------------------------------------ + da39a3ee5e6b4b0d3255bfef95601890afd80709 +(1 row) + +-- expected: da39a3ee5e6b4b0d3255bfef95601890afd80709 +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: dat3.aes.sha1.mdc.s2k3.z0 + +jA0EBwMCxQvxJZ3G/HRg0lgBeYmTa7/uDAjPyFwSX4CYBgpZWVn/JS8JzILrcWF8 +gFnkUKIE0PSaYFp+Yi1VlRfUtRQ/X/LYNGa7tWZS+4VQajz2Xtz4vUeAEiYFYPXk +73Hb8m1yRhQK +=ivrD +-----END PGP MESSAGE----- +'), '0123456789abcdefghij'), 'sha1'), 'hex'); + encode +------------------------------------------ + 5e5c135efc0dd00633efc6dfd6e731ea408a5b4c +(1 row) + +-- expected: 5e5c135efc0dd00633efc6dfd6e731ea408a5b4c +-- Checking CRLF +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: crlf mess + +ww0ECQMCt7VAtby6l4Bi0lgB5KMIZiiF/b3CfMfUyY0eDncsGXtkbu1X+l9brjpMP8eJnY79Amms +a3nsOzKTXUfS9VyaXo8IrncM6n7fdaXpwba/3tNsAhJG4lDv1k4g9v8Ix2dfv6Rs +=mBP9 +-----END PGP MESSAGE----- +'), 'key', 'convert-crlf=0'), 'sha1'), 'hex'); + encode +------------------------------------------ + 9353062be7720f1446d30b9e75573a4833886784 +(1 row) + +-- expected: 9353062be7720f1446d30b9e75573a4833886784 +select encode(digest(pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- +Comment: crlf mess + +ww0ECQMCt7VAtby6l4Bi0lgB5KMIZiiF/b3CfMfUyY0eDncsGXtkbu1X+l9brjpMP8eJnY79Amms +a3nsOzKTXUfS9VyaXo8IrncM6n7fdaXpwba/3tNsAhJG4lDv1k4g9v8Ix2dfv6Rs +=mBP9 +-----END PGP MESSAGE----- +'), 'key', 'convert-crlf=1'), 'sha1'), 'hex'); + encode +------------------------------------------ + 7efefcab38467f7484d6fa43dc86cf5281bd78e2 +(1 row) + +-- expected: 7efefcab38467f7484d6fa43dc86cf5281bd78e2 +-- check BUG #11905, problem with messages 6 less than a power of 2. +select pgp_sym_decrypt(pgp_sym_encrypt(repeat('x',65530),'1'),'1') = repeat('x',65530); + ?column? +---------- + t +(1 row) + +-- expected: true +-- Negative tests +-- Decryption with a certain incorrect key yields an apparent Literal Data +-- packet reporting its content to be binary data. Ciphertext source: +-- iterative pgp_sym_encrypt('secret', 'key') until the random prefix gave +-- rise to that property. +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +ww0EBwMCxf8PTrQBmJdl0jcB6y2joE7GSLKRv7trbNsF5Z8ou5NISLUg31llVH/S0B2wl4bvzZjV +VsxxqLSPzNLAeIspJk5G +=mSd/ +-----END PGP MESSAGE----- +'), 'wrong-key', 'debug=1'); +NOTICE: dbg: prefix_init: corrupt prefix +NOTICE: dbg: parse_literal_data: data type=b +NOTICE: dbg: mdcbuf_finish: bad MDC pkt hdr +ERROR: Wrong key or corrupt data +-- Routine text/binary mismatch. +select pgp_sym_decrypt(pgp_sym_encrypt_bytea('P', 'key'), 'key', 'debug=1'); +NOTICE: dbg: parse_literal_data: data type=b +ERROR: Not text data +-- Decryption with a certain incorrect key yields an apparent BZip2-compressed +-- plaintext. Ciphertext source: iterative pgp_sym_encrypt('secret', 'key') +-- until the random prefix gave rise to that property. +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +ww0EBwMC9rK/dMkF5Zlt0jcBlzAQ1mQY2qYbKYbw8h3EZ5Jk0K2IiY92R82TRhWzBIF/8cmXDPtP +GXsd65oYJZp3Khz0qfyn +=Nmpq +-----END PGP MESSAGE----- +'), 'wrong-key', 'debug=1'); +NOTICE: dbg: prefix_init: corrupt prefix +NOTICE: dbg: parse_compressed_data: bzip2 unsupported +NOTICE: dbg: mdcbuf_finish: bad MDC pkt hdr +ERROR: Wrong key or corrupt data +-- Routine use of BZip2 compression. Ciphertext source: +-- echo x | gpg --homedir /nonexistent --personal-compress-preferences bzip2 \ +-- --personal-cipher-preferences aes --no-emit-version --batch \ +-- --symmetric --passphrase key --armor +select pgp_sym_decrypt(dearmor(' +-----BEGIN PGP MESSAGE----- + +jA0EBwMCRhFrAKNcLVJg0mMBLJG1cCASNk/x/3dt1zJ+2eo7jHfjgg3N6wpB3XIe +QCwkWJwlBG5pzbO5gu7xuPQN+TbPJ7aQ2sLx3bAHhtYb0i3vV9RO10Gw++yUyd4R +UCAAw2JRIISttRHMfDpDuZJpvYo= +=AZ9M +-----END PGP MESSAGE----- +'), 'key', 'debug=1'); +NOTICE: dbg: parse_compressed_data: bzip2 unsupported +ERROR: Unsupported compression algorithm diff --git a/contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out b/contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out new file mode 100644 index 000000000000..f41c6c9893ab --- /dev/null +++ b/contrib/pgcrypto/expected/pgp-pubkey-decrypt_1.out @@ -0,0 +1,652 @@ +-- +-- PGP Public Key Encryption +-- +-- As most of the low-level stuff is tested in symmetric key +-- tests, here's only public-key specific tests +create table keytbl ( + id int4, + name text, + pubkey text, + seckey text +); +create table encdata ( + id int4, + data text +); +insert into keytbl (id, name, pubkey, seckey) +values (1, 'elg1024', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQLQfRWxnYW1hbCAx +MDI0IDx0ZXN0QGV4YW1wbGUub3JnPoheBBMRAgAeBQJCyCFIAhsDBgsJCAcDAgMV +AgMDFgIBAh4BAheAAAoJEBwpvA0YF3NkOtsAniI9W2bC3CxARTpYrev7ihreDzFc +AJ9WYLQxDQAi5Ec9AQoodPkIagzZ4LkBDQRCyCFKEAQAh5SNbbJMAsJ+sQbcWEzd +ku8AdYB5zY7Qyf9EOvn0g39bzANhxmmb6gbRlQN0ioymlDwraTKUAfuCZgNcg/0P +sxFGb9nDcvjIV8qdVpnq1PuzMFuBbmGI6weg7Pj01dlPiO0wt1lLX+SubktqbYxI ++h31c3RDZqxj+KAgxR8YNGMAAwYD+wQs2He1Z5+p4OSgMERiNzF0acZUYmc0e+/9 +6gfL0ft3IP+SSFo6hEBrkKVhZKoPSSRr5KpNaEobhdxsnKjUaw/qyoaFcNMzb4sF +k8wq5UlCkR+h72u6hv8FuleCV8SJUT1U2JjtlXJR2Pey9ifh8rZfu57UbdwdHa0v +iWc4DilhiEkEGBECAAkFAkLIIUoCGwwACgkQHCm8DRgXc2TtrwCfdPom+HlNVE9F +ig3hGY1Rb4NEk1gAn1u9IuQB+BgDP40YHHz6bKWS/x80 +=RWci +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQG7BELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQAAAnj4i4st+s+C6 +WKTIDcL1Iy0Saq8lCp60H0VsZ2FtYWwgMTAyNCA8dGVzdEBleGFtcGxlLm9yZz6I +XgQTEQIAHgUCQsghSAIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRAcKbwNGBdz +ZDrbAJ9cp6AsjOhiLxwznsMJheGf4xkH8wCfUPjMCLm4tAEnyYn2hDNt7CB8B6Kd +ATEEQsghShAEAIeUjW2yTALCfrEG3FhM3ZLvAHWAec2O0Mn/RDr59IN/W8wDYcZp +m+oG0ZUDdIqMppQ8K2kylAH7gmYDXIP9D7MRRm/Zw3L4yFfKnVaZ6tT7szBbgW5h +iOsHoOz49NXZT4jtMLdZS1/krm5Lam2MSPod9XN0Q2asY/igIMUfGDRjAAMGA/sE +LNh3tWefqeDkoDBEYjcxdGnGVGJnNHvv/eoHy9H7dyD/kkhaOoRAa5ClYWSqD0kk +a+SqTWhKG4XcbJyo1GsP6sqGhXDTM2+LBZPMKuVJQpEfoe9ruob/BbpXglfEiVE9 +VNiY7ZVyUdj3svYn4fK2X7ue1G3cHR2tL4lnOA4pYQAA9030E4u2ZKOfJBpUM+EM +m9VmsGjaQZV4teB0R/q3W8sRIYhJBBgRAgAJBQJCyCFKAhsMAAoJEBwpvA0YF3Nk +7a8AniFFotw1x2X+oryu3Q3nNtmxoKHpAJ9HU7jw7ydg33dI9J8gVkrmsSZ2/w== +=nvqq +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (2, 'elg2048', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELIIgoRBAC1onBpxKYgDvrgCaUWPY34947X3ogxGOfCN0p6Eqrx+2PUhm4n +vFvmczpMT4iDc0mUO+iwnwsEkXQI1eC99g8c0jnZAvzJZ5miAHL8hukMAMfDkYke +5aVvcPPc8uPDlItpszGmH0rM0V9TIt/i9QEXetpyNWhk4jj5qnohYhLeZwCgkOdO +RFAdNi4vfFPivvtAp2ffjU8D/R3x/UJCvkzi7i9rQHGo313xxmQu5BuqIjANBUij +8IE7LRPI/Qhg2hYy3sTJwImDi7VkS+fuvNVk0d6MTWplAXYU96bn12JaD21R9sKl +Fzcc+0iZI1wYA1PczisUkoTISE+dQFUsoGHfpDLhoBuesXQrhBavI8t8VPd+nkdt +J+oKA/9iRQ87FzxdYTkh2drrv69FZHc3Frsjw9nPcBq/voAvXH0MRilqyCg7HpW/ +T9naeOERksa+Rj4R57IF1l4e5oiiGJo9QmaKZcsCsXrREJCycrlEtMqXfSPy+bi5 +0yDZE/Qm1dwu13+OXOsRvkoNYjO8Mzo9K8wU12hMqN0a2bu6a7QjRWxnYW1hbCAy +MDQ4IDx0ZXN0MjA0OEBleGFtcGxlLm9yZz6IXgQTEQIAHgUCQsgiCgIbAwYLCQgH +AwIDFQIDAxYCAQIeAQIXgAAKCRBI6c1W/qZo29PDAKCG724enIxRog1j+aeCp/uq +or6mbwCePuKy2/1kD1FvnhkZ/R5fpm+pdm25Ag0EQsgiIhAIAJI3Gb2Ehtz1taQ9 +AhPY4Avad2BsqD3S5X/R11Cm0KBE/04D29dxn3f8QfxDsexYvNIZjoJPBqqZ7iMX +MhoWyw8ZF5Zs1mLIjFGVorePrm94N3MNPWM7x9M36bHUjx0vCZKFIhcGY1g+htE/ +QweaJzNVeA5z4qZmik41FbQyQSyHa3bOkTZu++/U6ghP+iDp5UDBjMTkVyqITUVN +gC+MR+da/I60irBVhue7younh4ovF+CrVDQJC06HZl6CAJJyA81SmRfi+dmKbbjZ +LF6rhz0norPjISJvkIqvdtM4VPBKI5wpgwCzpEqjuiKrAVujRT68zvBvJ4aVqb11 +k5QdJscAAwUH/jVJh0HbWAoiFTe+NvohfrA8vPcD0rtU3Y+siiqrabotnxJd2NuC +bxghJYGfNtnx0KDjFbCRKJVeTFok4UnuVYhXdH/c6i0/rCTNdeW2D6pmR4GfBozR +Pw/ARf+jONawGLyUj7uq13iquwMSE7VyNuF3ycL2OxXjgOWMjkH8c+zfHHpjaZ0R +QsetMq/iNBWraayKZnWUd+eQqNzE+NUo7w1jAu7oDpy+8a1eipxzK+O0HfU5LTiF +Z1Oe4Um0P2l3Xtx8nEgj4vSeoEkl2qunfGW00ZMMTCWabg0ZgxPzMfMeIcm6525A +Yn2qL+X/qBJTInAl7/hgPz2D1Yd7d5/RdWaISQQYEQIACQUCQsgiIgIbDAAKCRBI +6c1W/qZo25ZSAJ98WTrtl2HiX8ZqZq95v1+9cHtZPQCfZDoWQPybkNescLmXC7q5 +1kNTmEU= +=8QM5 +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQG7BELIIgoRBAC1onBpxKYgDvrgCaUWPY34947X3ogxGOfCN0p6Eqrx+2PUhm4n +vFvmczpMT4iDc0mUO+iwnwsEkXQI1eC99g8c0jnZAvzJZ5miAHL8hukMAMfDkYke +5aVvcPPc8uPDlItpszGmH0rM0V9TIt/i9QEXetpyNWhk4jj5qnohYhLeZwCgkOdO +RFAdNi4vfFPivvtAp2ffjU8D/R3x/UJCvkzi7i9rQHGo313xxmQu5BuqIjANBUij +8IE7LRPI/Qhg2hYy3sTJwImDi7VkS+fuvNVk0d6MTWplAXYU96bn12JaD21R9sKl +Fzcc+0iZI1wYA1PczisUkoTISE+dQFUsoGHfpDLhoBuesXQrhBavI8t8VPd+nkdt +J+oKA/9iRQ87FzxdYTkh2drrv69FZHc3Frsjw9nPcBq/voAvXH0MRilqyCg7HpW/ +T9naeOERksa+Rj4R57IF1l4e5oiiGJo9QmaKZcsCsXrREJCycrlEtMqXfSPy+bi5 +0yDZE/Qm1dwu13+OXOsRvkoNYjO8Mzo9K8wU12hMqN0a2bu6awAAn2F+iNBElfJS +8azqO/kEiIfpqu6/DQG0I0VsZ2FtYWwgMjA0OCA8dGVzdDIwNDhAZXhhbXBsZS5v +cmc+iF0EExECAB4FAkLIIgoCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQSOnN +Vv6maNvTwwCYkpcJmpl3aHCQdGomz7dFohDgjgCgiThZt2xTEi6GhBB1vuhk+f55 +n3+dAj0EQsgiIhAIAJI3Gb2Ehtz1taQ9AhPY4Avad2BsqD3S5X/R11Cm0KBE/04D +29dxn3f8QfxDsexYvNIZjoJPBqqZ7iMXMhoWyw8ZF5Zs1mLIjFGVorePrm94N3MN +PWM7x9M36bHUjx0vCZKFIhcGY1g+htE/QweaJzNVeA5z4qZmik41FbQyQSyHa3bO +kTZu++/U6ghP+iDp5UDBjMTkVyqITUVNgC+MR+da/I60irBVhue7younh4ovF+Cr +VDQJC06HZl6CAJJyA81SmRfi+dmKbbjZLF6rhz0norPjISJvkIqvdtM4VPBKI5wp +gwCzpEqjuiKrAVujRT68zvBvJ4aVqb11k5QdJscAAwUH/jVJh0HbWAoiFTe+Nvoh +frA8vPcD0rtU3Y+siiqrabotnxJd2NuCbxghJYGfNtnx0KDjFbCRKJVeTFok4Unu +VYhXdH/c6i0/rCTNdeW2D6pmR4GfBozRPw/ARf+jONawGLyUj7uq13iquwMSE7Vy +NuF3ycL2OxXjgOWMjkH8c+zfHHpjaZ0RQsetMq/iNBWraayKZnWUd+eQqNzE+NUo +7w1jAu7oDpy+8a1eipxzK+O0HfU5LTiFZ1Oe4Um0P2l3Xtx8nEgj4vSeoEkl2qun +fGW00ZMMTCWabg0ZgxPzMfMeIcm6525AYn2qL+X/qBJTInAl7/hgPz2D1Yd7d5/R +dWYAAVQKFPXbRaxbdArwRVXMzSD3qj/+VwwhwEDt8zmBGnlBfwVdkjQQrDUMmV1S +EwyISQQYEQIACQUCQsgiIgIbDAAKCRBI6c1W/qZo25ZSAJ4sgUfHTVsG/x3p3fcM +3b5R86qKEACggYKSwPWCs0YVRHOWqZY0pnHtLH8= +=3Dgk +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (3, 'elg4096', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELII7wRBACFuaAvb11cIvjJK9LkZr4cYuYhLWh3DJdojNNnLNiym5OEksvY +05cw8OgqKtPzICU7o/mHXTWhzJYUt3i50/AeYygI8Q0uATS6RnDAKNlES1EMoHKz +2a5iFbYs4bm4IwlkvYd8uWjcu+U0YLbxir39u+anIc6eT+q3WiH/q3zDRwCgkT98 +cnIG8iO8PdwDSP8G4Lt6TYED/R45GvCzJ4onQALLE92KkLUz8aFWSl05r84kczEN +SxiP9Ss6m465RmwWHfwYAu4b+c4GeNyU8fIU2EM8cezchC+edEi3xu1s+pCV0Dk4 +18DGC8WKCICO30vBynuNmYg7W/7Zd4wtjss454fMW7+idVDNM701mmXBtI1nsBtG +7Z4tA/9FxjFbJK9jh24RewfjHpLYqcfCo2SsUjOwsnMZ5yg2yv9KyVVQhRqwmrqt +q8MRyjGmfoD9PPdCgvqgzy0hHvAHUtTm2zUczGTG+0g4hNIklxC/Mv6J4KE+NWTh +uB4acqofHyaw2WnKOuRUsoDi6rG5AyjNMyAK/vVcEGj7J1tk27QjRWxnYW1hbCA0 +MDk2IDx0ZXN0NDA5NkBleGFtcGxlLm9yZz6IXgQTEQIAHgUCQsgjvAIbAwYLCQgH +AwIDFQIDAxYCAQIeAQIXgAAKCRBj+HX2P2d0oAEDAJ9lI+CNmb42z3+a6TnVusM6 +FI7oLwCfUwA1zEcRdsT3nIkoYh0iKxFSDFW5BA0EQsgkdhAQAJQbLXlgcJ/jq+Xh +Eujb77/eeftFJObNIRYD9fmJ7HFIXbUcknEpbs+cRH/nrj5dGSY3OT3jCXOUtvec +sCoX/CpZWL0oqDjAiZtNSFiulw5Gav4gHYkWKgKdSo+2rkavEPqKIVHvMeXaJtGT +d7v/AmL/P8T7gls93o5WFBOLtPbDvWqaKRy2U5TAhl1laiM0vGALRVjvSCgnGw9g +FpSnXbO3AfenUSjDzZujfGLHtU44ixHSS/D4DepiF3YaYLsN4CBqZRv6FbMZD5W3 +DnJY4kS1kH0MzdcF19TlcZ3itTCcGIt1tMKf84mccPoqdMzH7vumBGTeFEly5Afp +9berJcirqh2fzlunN0GS02z6SGWnjTbDlkNDxuxPSBbpcpNyD3jpYAUqSwRsZ/+5 +zkzcbGtDmvy9sJ5lAXkxGoIoQ1tEVX/LOHnh2NQHK8ourVOnr7MS0nozssITZJ5E +XqtHiREjiYEuPyZiVZKJHLWuYYaF+n40znnz3sJuXFRreHhHbbvRdlYUU5mJV+XZ +BLgKuS33NdpGeMIngnCc/9IQ6OZb6ixc94kbkd3w2PVr8CbKlu/IHTjWOO2mAo+D ++OydlYl23FiM3KOyMP1HcEOJMB/nwkMtrvd+522Lu9n77ktKfot9IPrQDIQTyXjR +3pCOFtCOBnk2tJHMPoG9jn9ah/LHAAMHEACDZ5I/MHGfmiKg2hrmqBu2J2j/deC8 +CpwcyDH1ovQ0gHvb9ESa+CVRU2Wdy2CD7Q9SmtMverB5eneL418iPVRcQdwRmQ2y +IH4udlBa6ce9HTUCaecAZ4/tYBnaC0Av/9l9tz14eYcwRMDpB+bnkhgF+PZ1KAfD +9wcY2aHbtsf3lZBc5h4owPJkxpe/BNzuJxW3q4VpSbLsZhwnCZ2wg7DRwP44wFIk +00ptmoBY59gsU6I40XtzrF8JDr0cA57xND5RY21Z8lnnYRE1Tc8h5REps9ZIxW3/ +yl91404bPLqxczpUHQAMSTAmBaStPYX1nS51uofOhLs5SKPCUmxfGKIOhsD0oLUn +78DnkONVGeXzBibSwwtbgfMzee4G8wSUfJ7w8WXz1TyanaGLnJ+DuKASSOrFoBCD +HEDuWZWgSL74NOQupFRk0gxOPmqU94Y8HziQWma/cETbmD83q8rxN+GM2oBxQkQG +xcbqMTHE7aVhV3tymbSWVaYhww3oIwsZS9oUIi1DnPEowS6CpVRrwdvLjLJnJzzV +O3AFPn9eZ1Q7R1tNx+zZ4OOfhvI/OlRJ3HBx2L53embkbdY9gFYCCdTjPyjKoDIx +kALgCajjCYMNUsAKNSd6mMCQ8TtvukSzkZS1RGKP27ohsdnzIVsiEAbxDMMcI4k1 +ul0LExUTCXSjeIhJBBgRAgAJBQJCyCR2AhsMAAoJEGP4dfY/Z3Sg19sAn0NDS8pb +qrMpQAxSb7zRTmcXEFd9AJ435H0ttP/NhLHXC9ezgbCMmpXMOQ== +=kRxT +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQG7BELII7wRBACFuaAvb11cIvjJK9LkZr4cYuYhLWh3DJdojNNnLNiym5OEksvY +05cw8OgqKtPzICU7o/mHXTWhzJYUt3i50/AeYygI8Q0uATS6RnDAKNlES1EMoHKz +2a5iFbYs4bm4IwlkvYd8uWjcu+U0YLbxir39u+anIc6eT+q3WiH/q3zDRwCgkT98 +cnIG8iO8PdwDSP8G4Lt6TYED/R45GvCzJ4onQALLE92KkLUz8aFWSl05r84kczEN +SxiP9Ss6m465RmwWHfwYAu4b+c4GeNyU8fIU2EM8cezchC+edEi3xu1s+pCV0Dk4 +18DGC8WKCICO30vBynuNmYg7W/7Zd4wtjss454fMW7+idVDNM701mmXBtI1nsBtG +7Z4tA/9FxjFbJK9jh24RewfjHpLYqcfCo2SsUjOwsnMZ5yg2yv9KyVVQhRqwmrqt +q8MRyjGmfoD9PPdCgvqgzy0hHvAHUtTm2zUczGTG+0g4hNIklxC/Mv6J4KE+NWTh +uB4acqofHyaw2WnKOuRUsoDi6rG5AyjNMyAK/vVcEGj7J1tk2wAAoJCUNy6awTkw +XfbLbpqh0fvDst7jDLa0I0VsZ2FtYWwgNDA5NiA8dGVzdDQwOTZAZXhhbXBsZS5v +cmc+iF4EExECAB4FAkLII7wCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQY/h1 +9j9ndKABAwCeNEOVK87EzXYbtxYBsnjrUI948NIAn2+f3BXiBFDV5NvqPwIZ0m77 +Fwy4nQRMBELIJHYQEACUGy15YHCf46vl4RLo2++/3nn7RSTmzSEWA/X5iexxSF21 +HJJxKW7PnER/564+XRkmNzk94wlzlLb3nLAqF/wqWVi9KKg4wImbTUhYrpcORmr+ +IB2JFioCnUqPtq5GrxD6iiFR7zHl2ibRk3e7/wJi/z/E+4JbPd6OVhQTi7T2w71q +mikctlOUwIZdZWojNLxgC0VY70goJxsPYBaUp12ztwH3p1Eow82bo3xix7VOOIsR +0kvw+A3qYhd2GmC7DeAgamUb+hWzGQ+Vtw5yWOJEtZB9DM3XBdfU5XGd4rUwnBiL +dbTCn/OJnHD6KnTMx+77pgRk3hRJcuQH6fW3qyXIq6odn85bpzdBktNs+khlp402 +w5ZDQ8bsT0gW6XKTcg946WAFKksEbGf/uc5M3GxrQ5r8vbCeZQF5MRqCKENbRFV/ +yzh54djUByvKLq1Tp6+zEtJ6M7LCE2SeRF6rR4kRI4mBLj8mYlWSiRy1rmGGhfp+ +NM55897CblxUa3h4R2270XZWFFOZiVfl2QS4Crkt9zXaRnjCJ4JwnP/SEOjmW+os +XPeJG5Hd8Nj1a/AmypbvyB041jjtpgKPg/jsnZWJdtxYjNyjsjD9R3BDiTAf58JD +La73fudti7vZ++5LSn6LfSD60AyEE8l40d6QjhbQjgZ5NrSRzD6BvY5/WofyxwAD +BxAAg2eSPzBxn5oioNoa5qgbtido/3XgvAqcHMgx9aL0NIB72/REmvglUVNlnctg +g+0PUprTL3qweXp3i+NfIj1UXEHcEZkNsiB+LnZQWunHvR01AmnnAGeP7WAZ2gtA +L//Zfbc9eHmHMETA6Qfm55IYBfj2dSgHw/cHGNmh27bH95WQXOYeKMDyZMaXvwTc +7icVt6uFaUmy7GYcJwmdsIOw0cD+OMBSJNNKbZqAWOfYLFOiONF7c6xfCQ69HAOe +8TQ+UWNtWfJZ52ERNU3PIeURKbPWSMVt/8pfdeNOGzy6sXM6VB0ADEkwJgWkrT2F +9Z0udbqHzoS7OUijwlJsXxiiDobA9KC1J+/A55DjVRnl8wYm0sMLW4HzM3nuBvME +lHye8PFl89U8mp2hi5yfg7igEkjqxaAQgxxA7lmVoEi++DTkLqRUZNIMTj5qlPeG +PB84kFpmv3BE25g/N6vK8TfhjNqAcUJEBsXG6jExxO2lYVd7cpm0llWmIcMN6CML +GUvaFCItQ5zxKMEugqVUa8Hby4yyZyc81TtwBT5/XmdUO0dbTcfs2eDjn4byPzpU +Sdxwcdi+d3pm5G3WPYBWAgnU4z8oyqAyMZAC4Amo4wmDDVLACjUnepjAkPE7b7pE +s5GUtURij9u6IbHZ8yFbIhAG8QzDHCOJNbpdCxMVEwl0o3gAAckBdfKuasiNUn5G +L5XRnSvaOFzftr8zteOlZChCSNvzH5k+i1j7RJbWq06OeKRywPzjfjgM2MvRzI43 +ICeISQQYEQIACQUCQsgkdgIbDAAKCRBj+HX2P2d0oNfbAJ9+G3SeXrk+dWwo9EGi +hqMi2GVTsgCfeoQJPsc8FLYUgfymc/3xqAVLUtg= +=Gjq6 +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (4, 'rsa2048', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQELBELIJbEBCADAIdtcoLAmQfl8pb73pPRuEYx8qW9klLfCGG5A4OUOi00JHNwP +ZaABe1PGzjoeXrgM1MTQZhoZu1Vdg+KDI6XAtiy9P6bLg7ntsXksD4wBoIKtQKc2 +55pdukxTiu+xeJJG2q8ZZPOp97CV9fbQ9vPCwgnuSsDCoQlibZikDVPAyVTvp7Jx +5rz8yXsl4sxvaeMZPqqFPtA/ENeQ3cpsyR1BQXSvoZpH1Fq0b8GcZTEdWWD/w6/K +MCRC8TmgEd+z3e8kIsCwFQ+TSHbCcxRWdgZE7gE31sJHHVkrZlXtLU8MPXWqslVz +R0cX+yC8j6bXI6/BqZ2SvRndJwuunRAr4um7AAYptB5SU0EgMjA0OCA8cnNhMjA0 +OEBleGFtcGxlLm9yZz6JATQEEwECAB4FAkLIJbECGwMGCwkIBwMCAxUCAwMWAgEC +HgECF4AACgkQnc+OnJvTHyQqHwf8DtzuAGmObfe3ggtn14x2wnU1Nigebe1K5liR +nrLuVlLBpdO6CWmMUzfKRvyZlx54GlA9uUQSjW+RlgejdOTQqesDrcTEukYd4yzw +bLZyM5Gb3lsE/FEmE7Dxw/0Utf59uACqzG8LACQn9J6sEgZWKxAupuYTHXd12lDP +D3dnU4uzKPhMcjnSN00pzjusP7C9NZd3OLkAx2vw/dmb4Q+/QxeZhVYYsAUuR2hv +9bgGWopumlOkt8Zu5YG6+CtTbJXprPI7pJ1jHbeE+q/29hWJQtS8Abx82AcOkzhv +S3NZKoJ/1DrGgoDAu1mGkM4KvLAxfDs/qQ9dZhtEmDbKPLTVEA== +=lR4n +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQOWBELIJbEBCADAIdtcoLAmQfl8pb73pPRuEYx8qW9klLfCGG5A4OUOi00JHNwP +ZaABe1PGzjoeXrgM1MTQZhoZu1Vdg+KDI6XAtiy9P6bLg7ntsXksD4wBoIKtQKc2 +55pdukxTiu+xeJJG2q8ZZPOp97CV9fbQ9vPCwgnuSsDCoQlibZikDVPAyVTvp7Jx +5rz8yXsl4sxvaeMZPqqFPtA/ENeQ3cpsyR1BQXSvoZpH1Fq0b8GcZTEdWWD/w6/K +MCRC8TmgEd+z3e8kIsCwFQ+TSHbCcxRWdgZE7gE31sJHHVkrZlXtLU8MPXWqslVz +R0cX+yC8j6bXI6/BqZ2SvRndJwuunRAr4um7AAYpAAf/QZsrrz0c7dgWwGqMIpw6 +fP+/lLa74+fa2CFRWtYowEiKsfDg/wN7Ua07036dNhPa8aZPsU6SRzm5PybKOURe +D9pNt0FxJkX0j5pCWfjSJgTbc1rCdqZ/oyBk/U6pQtf//zfw3PbDl7I8TC6GOt2w +5NgcXdsWHP7LAmPctOVUyzFsenevR0MFTHkMbmKI1HpFm8XN/e1Fl+qIAD+OagTF +5B32VvpoJtkh5nxnIuToNJsa9Iy7F9MM2CeFOyTMihMcjXKBBUaAYoF115irBvqu +7N/qWmzqLg8yxBZ56mh6meCF3+67VA2y7fL8rhw2QuqgLg1JFlKAVL+9crCSrn// +GQQA1kT7FytW6BNOffblFYZkrJer3icoRDqa/ljgH/yVaWoVT1igy0E9XzYO7MwP +2usj/resLy0NC1qCthk51cZ/wthooMl88e5Wb4l5FYwBEac7muSBTo4W8cAH1hFj +TWL6XAGvEzGX3Mt9pn8uYGlQLZAhJoNCAU2EOCbN1PchDvsEAOWNKYesuUVk8+sQ +St0NDNhd9BWtTWTHkCZb1dKC3JTfr9PqkTBLrWFbYjkOtvdPAW7FDaXXXZfdH1jH +WfwP3Q+I6sqgSaWpCS4dBAns3/RVtO7czVgyIwma04iIvJqderYrfvkUq95KfwP2 +V8wXkhrPPPxyrg5y3wQlpY2jb5RBBAC17SK1ms+DBtck4vpdjp3SJ32SbyC/DU30 +89Q12j74S7Zdu1qZlKnvy3kWPYX/hMuSzGZ+mLVJNFEqH2X01aFzppYz0hdI9PGB +9tTFEqZWQL9ZkXfjc79Cgnt12pNukRbtw0N/kyutOdIFHVT79wVAd+powqziXJsC +Kc+4xjwSCkZitB5SU0EgMjA0OCA8cnNhMjA0OEBleGFtcGxlLm9yZz6JATQEEwEC +AB4FAkLIJbECGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQnc+OnJvTHyQqHwf8 +DtzuAGmObfe3ggtn14x2wnU1Nigebe1K5liRnrLuVlLBpdO6CWmMUzfKRvyZlx54 +GlA9uUQSjW+RlgejdOTQqesDrcTEukYd4yzwbLZyM5Gb3lsE/FEmE7Dxw/0Utf59 +uACqzG8LACQn9J6sEgZWKxAupuYTHXd12lDPD3dnU4uzKPhMcjnSN00pzjusP7C9 +NZd3OLkAx2vw/dmb4Q+/QxeZhVYYsAUuR2hv9bgGWopumlOkt8Zu5YG6+CtTbJXp +rPI7pJ1jHbeE+q/29hWJQtS8Abx82AcOkzhvS3NZKoJ/1DrGgoDAu1mGkM4KvLAx +fDs/qQ9dZhtEmDbKPLTVEA== +=WKAv +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (5, 'psw-elg1024', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQGiBELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQLQfRWxnYW1hbCAx +MDI0IDx0ZXN0QGV4YW1wbGUub3JnPoheBBMRAgAeBQJCyCFIAhsDBgsJCAcDAgMV +AgMDFgIBAh4BAheAAAoJEBwpvA0YF3NkOtsAniI9W2bC3CxARTpYrev7ihreDzFc +AJ9WYLQxDQAi5Ec9AQoodPkIagzZ4LkBDQRCyCFKEAQAh5SNbbJMAsJ+sQbcWEzd +ku8AdYB5zY7Qyf9EOvn0g39bzANhxmmb6gbRlQN0ioymlDwraTKUAfuCZgNcg/0P +sxFGb9nDcvjIV8qdVpnq1PuzMFuBbmGI6weg7Pj01dlPiO0wt1lLX+SubktqbYxI ++h31c3RDZqxj+KAgxR8YNGMAAwYD+wQs2He1Z5+p4OSgMERiNzF0acZUYmc0e+/9 +6gfL0ft3IP+SSFo6hEBrkKVhZKoPSSRr5KpNaEobhdxsnKjUaw/qyoaFcNMzb4sF +k8wq5UlCkR+h72u6hv8FuleCV8SJUT1U2JjtlXJR2Pey9ifh8rZfu57UbdwdHa0v +iWc4DilhiEkEGBECAAkFAkLIIUoCGwwACgkQHCm8DRgXc2TtrwCfdPom+HlNVE9F +ig3hGY1Rb4NEk1gAn1u9IuQB+BgDP40YHHz6bKWS/x80 +=RWci +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQHpBELIIUgRBACp401L6jXrLB28c3YA4sM3OJKnxM1GT9YTkWyE3Vyte65H8WU9 +tGPBX7OMuaX5eGZ84LFUGvaP0k7anfmXcDkCO3P9GgL+ro/dS2Ps/vChQPZqHaxE +xpKDUt47B7DGdRJrC8DRnIR4wbSyQA6ma3S1yFqC5pJhSs+mqf9eExOjiwCgntth +klRxIYw352ZX9Ov9oht/p/ED/1Xi4PS+tkXVvyIw5aZfa61bT6XvDkoPI0Aj3GE5 +YmCHJlKA/IhEr8QJOLV++5VEv4l6KQ1/DFoJzoNdr1AGJukgTc6X/WcQRzfQtUic +PHQme5oAWoHa6bVQZOwvbJh3mOXDq/Tk/KF22go8maM44vMn4bvv+SBbslviYLiL +jZJ1A/9JXF1esNq+X9HehJyqHHU7LEEf/ck6zC7o2erM3/LZlZuLNPD2cv3oL3Nv +saEgcTSZl+8XmO8pLmzjKIb+hi70qVx3t2IhMqbb4B/dMY1Ck62gPBKa81/Wwi7v +IsEBQLEtyBmGmI64YpzoRNFeaaF9JY+sAKqROqe6dLjJ7vebQP4HAwImKZ5q2QwT +D2DDAY/IQBjes7WgqZeacfLPDoB8ecD/KLoSCH6Z3etvbPHSOKiazxoJ962Ix74H +ZAE6ZbMTtl5dZW1ptB9FbGdhbWFsIDEwMjQgPHRlc3RAZXhhbXBsZS5vcmc+iF4E +ExECAB4FAkLIIUgCGwMGCwkIBwMCAxUCAwMWAgECHgECF4AACgkQHCm8DRgXc2Q6 +2wCfXKegLIzoYi8cM57DCYXhn+MZB/MAn1D4zAi5uLQBJ8mJ9oQzbewgfAeinQFf +BELIIUoQBACHlI1tskwCwn6xBtxYTN2S7wB1gHnNjtDJ/0Q6+fSDf1vMA2HGaZvq +BtGVA3SKjKaUPCtpMpQB+4JmA1yD/Q+zEUZv2cNy+MhXyp1WmerU+7MwW4FuYYjr +B6Ds+PTV2U+I7TC3WUtf5K5uS2ptjEj6HfVzdENmrGP4oCDFHxg0YwADBgP7BCzY +d7Vnn6ng5KAwRGI3MXRpxlRiZzR77/3qB8vR+3cg/5JIWjqEQGuQpWFkqg9JJGvk +qk1oShuF3GycqNRrD+rKhoVw0zNviwWTzCrlSUKRH6Hva7qG/wW6V4JXxIlRPVTY +mO2VclHY97L2J+Hytl+7ntRt3B0drS+JZzgOKWH+BwMCJimeatkMEw9gRkFjt4Xa +9rX8awMBE5+vVcGKv/DNiCvJnlYvSdCj8VfuHsYFliiJo6u17NJon+K43e3yvDNk +f631VOVanGEz7TyqOkWQiEkEGBECAAkFAkLIIUoCGwwACgkQHCm8DRgXc2TtrwCe +IUWi3DXHZf6ivK7dDec22bGgoekAn0dTuPDvJ2Dfd0j0nyBWSuaxJnb/ +=SNvr +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (6, 'rsaenc2048', ' +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +mQELBELr2m0BCADOrnknlnXI0EzRExf/TgoHvK7Xx/E0keWqV3KrOyC3/tY2KOrj +UVxaAX5pkFX9wdQObGPIJm06u6D16CH6CildX/vxG7YgvvKzK8JGAbwrXAfk7OIW +czO2zRaZGDynoK3mAxHRBReyTKtNv8rDQhuZs6AOozJNARdbyUO/yqUnqNNygWuT +4htFDEuLPIJwAbMSD0BvFW6YQaPdxzaAZm3EWVNbwDzjgbBUdBiUUwRdZIFUhsjJ +dirFdy5+uuZru6y6CNC1OERkJ7P8EyoFiZckAIE5gshVZzNuyLOZjc5DhWBvLbX4 +NZElAnfiv+4nA6y8wQLSIbmHA3nqJaBklj85AAYptCVSU0EgMjA0OCBFbmMgPHJz +YTIwNDhlbmNAZXhhbXBsZS5vcmc+iQE0BBMBAgAeBQJC69ptAhsDBgsJCAcDAgMV +AgMDFgIBAh4BAheAAAoJEMiZ6pNEGVVZHMkIAJtGHHZ9iM8Yq1rr0zl1L6SvlQP8 +JCaxHa31wH3PKqGtq2M+cpb2rXf7gAY/doHJPXggfVzkyFrysmQ1gPbDGYLyOutw ++IkhihEb5bWxQBNj+3zAFs1YX6v2HXWbSUSmyY1V9/+NTtKk03olDc/swd3lXzku +UOhcgfpBgIt3Q+MpT6M2+OIF7lVfSb1rWdpwTfGhZzW9szQOeoS4gPvxCCRyuabQ +RJ6DWH61F8fFIDJg1z+A/Obx4fqX6GOA69RzgZ3oukFBIXxNwV9PZNnAmHtZVYO8 +0g/oVYBbuvOYedffDBeQarhERZ5W2TnIE+nqY61YOLBqosliygdZTXULzNi5AQsE +QuvaugEIAOuCJZdkzORA6e1lr81Lnr4JzMsVBFA+X/yIkBbV6qX/A4nVSLAZKNPX +z1YIrMTu+1rMIiy10IWbA6zgMTpzPhJRfgePONgdnCYyK5Ksh5/C5ntzKwwGwxfK +lAXIxJurCHXTbEa+YvPdn76vJ3HsXOXVEL+fLb4U3l3Ng87YM202Lh1Ha2MeS2zE +FZcAoKbFqAAjDLEai64SoOFh0W3CsD1DL4zmfp+YZrUPHTtZadsi53i4KKW/ws9U +rHlolqYNhYze/uRLyfnUx9PN4r/GhEzauyDMV0smo91uB3aewPft+eCpmeWnu0PF +JVK4xyRmhIq2rVCw16a1pBJirvGM+y0ABimJAR8EGAECAAkFAkLr2roCGwwACgkQ +yJnqk0QZVVku1wgAg1bLSjPkhw+ldG5HzumpqR84+JKyozdJaJzefu2+1iqYE0B0 +WLz2PJVIiK41xiEkKhBvTOQYuXmtWqAWXptD91P5SoXoNJWLQO3TNwarANhHxkWg +w/TOUxQqoctlRUej5NDD+4eW5G9lcS1FEGuKDWtX096u80vO+TbyJjvx2eVM1k+X +dmeYsGOiNgDimCreJGYc14G7eY9jt24gw10n1sMAKI1qm6lcoHqZ9OOyla+wJdro +PYZGO7R8+1O9R22WrK6BYDT5j/1JwMZqbOESjNvDEVT0yOHClCHRN4CChbt6LhKh +CLUNdz/udIt0JAC6c/HdPLSW3HnmM3+iNj+Kug== +=pwU2 +-----END PGP PUBLIC KEY BLOCK----- +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.1 (GNU/Linux) + +lQOWBELr2m0BCADOrnknlnXI0EzRExf/TgoHvK7Xx/E0keWqV3KrOyC3/tY2KOrj +UVxaAX5pkFX9wdQObGPIJm06u6D16CH6CildX/vxG7YgvvKzK8JGAbwrXAfk7OIW +czO2zRaZGDynoK3mAxHRBReyTKtNv8rDQhuZs6AOozJNARdbyUO/yqUnqNNygWuT +4htFDEuLPIJwAbMSD0BvFW6YQaPdxzaAZm3EWVNbwDzjgbBUdBiUUwRdZIFUhsjJ +dirFdy5+uuZru6y6CNC1OERkJ7P8EyoFiZckAIE5gshVZzNuyLOZjc5DhWBvLbX4 +NZElAnfiv+4nA6y8wQLSIbmHA3nqJaBklj85AAYpAAf9GuKpxrXp267eSPw9ZeSw +Ik6ob1I0MHbhhHeaXQnF0SuOViJ1+Bs74hUB3/F5fqrnjVLIS/ysYzegYpbpXOIa +MZwYcp2e+dpmVb7tkGQgzXH0igGtBQBqoSUVq9mG2XKPVh2JmiYgOH6GrHSGmnCq +GCgEK4ezSomB/3OtPFSjAxOlSw6dXSkapSxW3pEGvCdaWd9p8yl4rSpGsZEErPPL +uSbZZrHtWfgq5UXdPeE1UnMlBcvSruvpN4qgWMgSMs4d2lXvzXJLcht/nryP+atT +H1gwnRmlDCVv5BeJepKo3ORJDvcPlXkJPhqS9If3BhTqt6QgQEFI4aIYYZOZpZoi +2QQA2Zckzktmsc1MS04zS9gm1CbxM9d2KK8EOlh7fycRQhYYqqavhTBH2MgEp+Dd +ZtuEN5saNDe9x/fwi2ok1Bq6luGMWPZU/nZe7fxadzwfliy/qPzStWFW3vY9mMLu +6uEqgjin/lf4YrAswXDZaEc5e4GuNgGfwr27hpjxE1jg3PsEAPMqXEOMT2yh+yRu +DlLRbFhYOI4aUHY2CGoQQONnwv2O5gFvmOcPlg3J5lvnwlOYCx0c3bDxAtHyjPJq +FAZqcJBaB9RDhKHwlWDrbx/6FPH2SuKE+u4msIhPFin4V3FAP+yTem/TKrdnaWy6 +EUrhCWTXVRTijBaCudfjFd/ipHZbA/0dv7UAcoWK6kiVLzyE+jOvtN+ZxTzxq7CW +mlFPgAC966hgJmz9IXqadtMgPAoL3PK9q1DbPM3JhsQcJrNzTJqZrdN1/kPU0HHa ++aof1BVy3wSvp2mXgaRUULStyhUIyBRM6hAYp3/MoWEYn/bwr+zQkIU8Zsk6OsZ6 +q1xE3cowrUWFtCVSU0EgMjA0OCBFbmMgPHJzYTIwNDhlbmNAZXhhbXBsZS5vcmc+ +iQE0BBMBAgAeBQJC69ptAhsDBgsJCAcDAgMVAgMDFgIBAh4BAheAAAoJEMiZ6pNE +GVVZHMkIAJtGHHZ9iM8Yq1rr0zl1L6SvlQP8JCaxHa31wH3PKqGtq2M+cpb2rXf7 +gAY/doHJPXggfVzkyFrysmQ1gPbDGYLyOutw+IkhihEb5bWxQBNj+3zAFs1YX6v2 +HXWbSUSmyY1V9/+NTtKk03olDc/swd3lXzkuUOhcgfpBgIt3Q+MpT6M2+OIF7lVf +Sb1rWdpwTfGhZzW9szQOeoS4gPvxCCRyuabQRJ6DWH61F8fFIDJg1z+A/Obx4fqX +6GOA69RzgZ3oukFBIXxNwV9PZNnAmHtZVYO80g/oVYBbuvOYedffDBeQarhERZ5W +2TnIE+nqY61YOLBqosliygdZTXULzNidA5YEQuvaugEIAOuCJZdkzORA6e1lr81L +nr4JzMsVBFA+X/yIkBbV6qX/A4nVSLAZKNPXz1YIrMTu+1rMIiy10IWbA6zgMTpz +PhJRfgePONgdnCYyK5Ksh5/C5ntzKwwGwxfKlAXIxJurCHXTbEa+YvPdn76vJ3Hs +XOXVEL+fLb4U3l3Ng87YM202Lh1Ha2MeS2zEFZcAoKbFqAAjDLEai64SoOFh0W3C +sD1DL4zmfp+YZrUPHTtZadsi53i4KKW/ws9UrHlolqYNhYze/uRLyfnUx9PN4r/G +hEzauyDMV0smo91uB3aewPft+eCpmeWnu0PFJVK4xyRmhIq2rVCw16a1pBJirvGM ++y0ABikAB/oC3z7lv6sVg+ngjbpWy9lZu2/ECZ9FqViVz7bUkjfvSuowgpncryLW +4EpVV4U6mMSgU6kAi5VGT/BvYGSAtnqDWGiPs7Kk+h4Adz74bEAXzU280pNBtSfX +tGvzlS4a376KzYFSCJDRBdMebEhJMbY0wQmR8lTZu5JSUI4YYEuN0c7ckdsw8w42 +QWTLonG8HC6h8UPKS0EAcaCo7tFubMIesU6cWuTYucsHE+wjbADjuSNX968qczNe +NoL2BUznXOQoPu6HQO4/8cr7ib+VQkB2bHQcMoZazPUStIID1e4CL4XcxfuAmT8o +3XDvMLgVqNp5W2f8Mzmk3/DbtsLXLOv5BADsCzQpseC8ikSYJC72hcon1wlUmGeH +3qgGiiHhYXFa18xgI5juoO8DaWno0rPPlgr36Y8mSB5qjYHMXwjKnKyUmt11H+hU ++6uk4hq3Rjd8l+vfuOSr1xoTrtBUg9Rwfw6JVo0DC+8CWg4oBWsLXVM6KQXPFdJs +8kyFQplR/iP1XQQA/2tbDANjAYGNNDjJO9/0kEnSAUyYMasFJDrA2q17J5CroVQw +QpMmWwdDkRANUVPKnWHS5sS65BRc7UytKe2f3A3ZInGXJIK2Hl+TzapWYcYxql+4 +ol5mEDDMDbhEE8Wmj9KyB6iifdLI0K+yxNb9T4Jpj3J18+St+G8+9AcFcBEEAM1b +M9C+/05cnV8gjcByqH9M9ypo8fzPvMKVXWwCLQXpaL50QIkzLURkiMoEWrCdELaA +sVPotRzePTIQ1ooLeDxd1gRnDqjZiIR0kwmv6vq8tfzY96O2ZbGWFI5eth89aWEJ +WB8AR3zYcXpwJLwPuhXW2/NlZF0bclJ3jNzAfTIeQmeJAR8EGAECAAkFAkLr2roC +GwwACgkQyJnqk0QZVVku1wgAg1bLSjPkhw+ldG5HzumpqR84+JKyozdJaJzefu2+ +1iqYE0B0WLz2PJVIiK41xiEkKhBvTOQYuXmtWqAWXptD91P5SoXoNJWLQO3TNwar +ANhHxkWgw/TOUxQqoctlRUej5NDD+4eW5G9lcS1FEGuKDWtX096u80vO+TbyJjvx +2eVM1k+XdmeYsGOiNgDimCreJGYc14G7eY9jt24gw10n1sMAKI1qm6lcoHqZ9OOy +la+wJdroPYZGO7R8+1O9R22WrK6BYDT5j/1JwMZqbOESjNvDEVT0yOHClCHRN4CC +hbt6LhKhCLUNdz/udIt0JAC6c/HdPLSW3HnmM3+iNj+Kug== +=UKh3 +-----END PGP PRIVATE KEY BLOCK----- +'); +insert into keytbl (id, name, pubkey, seckey) +values (7, 'rsaenc2048-psw', ' +same key with password +', ' +-----BEGIN PGP PRIVATE KEY BLOCK----- +Version: GnuPG v1.4.11 (GNU/Linux) + +lQPEBELr2m0BCADOrnknlnXI0EzRExf/TgoHvK7Xx/E0keWqV3KrOyC3/tY2KOrj +UVxaAX5pkFX9wdQObGPIJm06u6D16CH6CildX/vxG7YgvvKzK8JGAbwrXAfk7OIW +czO2zRaZGDynoK3mAxHRBReyTKtNv8rDQhuZs6AOozJNARdbyUO/yqUnqNNygWuT +4htFDEuLPIJwAbMSD0BvFW6YQaPdxzaAZm3EWVNbwDzjgbBUdBiUUwRdZIFUhsjJ +dirFdy5+uuZru6y6CNC1OERkJ7P8EyoFiZckAIE5gshVZzNuyLOZjc5DhWBvLbX4 +NZElAnfiv+4nA6y8wQLSIbmHA3nqJaBklj85AAYp/gcDCNnoEKwFo86JYCE1J92R +HRQ7DoyAZpW1O0dTXL8Epk0sKsKDrCJOrIkDymsjfyBexADIeqOkioy/50wD2Mku +CVHKWO2duAiJN5t/FoRgpR1/Q11K6QdfqOG0HxwfIXLcPv7eSIso8kWorj+I01BP +Fn/atGEbIjdWaz/q2XHbu0Q3x6Et2gIsbLRVMhiYz1UG9uzGJ0TYCdBa2SFhs184 +52akMpD+XVdM0Sq9/Cx40Seo8hzERB96+GXnQ48q2OhlvcEXiFyD6M6wYCWbEV+6 +XQVMymbl22FPP/bD9ReQX2kjrkQlFAtmhr+0y8reMCbcxwLuQfA3173lSPo7jrbH +oLrGhkRpqd2bYCelqdy/XMmRFso0+7uytHfTFrUNfDWfmHVrygoVrNnarCbxMMI0 +I8Q+tKHMThWgf0rIOSh0+w38kOXFCEqEWF8YkAqCrMZIlJIed78rOCFgG4aHajZR +D8rpXdUOIr/WeUddK25Tu8IuNJb0kFf12IMgNh0nS+mzlqWiofS5kA0TeB8wBV6t +RotaeyDNSsMoowfN8cf1yHMTxli+K1Tasg003WVUoWgUc+EsJ5+KTNwaX5uGv0Cs +j6dg6/FVeVRL9UsyF+2kt7euX3mABuUtcVGx/ZKTq/MNGEh6/r3B5U37qt+FDRbw +ppKPc2AP+yBUWsQskyrxFgv4eSpcLEg+lgdz/zLyG4qW4lrFUoO790Cm/J6C7/WQ +Z+E8kcS8aINJkg1skahH31d59ZkbW9PVeJMFGzNb0Z2LowngNP/BMrJ0LT2CQyLs +UxbT16S/gwAyUpJnbhWYr3nDdlwtC0rVopVTPD7khPRppcsq1f8D70rdIxI4Ouuw +vbjNZ1EWRJ9f2Ywb++k/xgSXwJkGodUlrUr+3i8cv8mPx+fWvif9q7Y5Ex1wCRa8 +8FAj/o+hEbQlUlNBIDIwNDggRW5jIDxyc2EyMDQ4ZW5jQGV4YW1wbGUub3JnPokB +NAQTAQIAHgUCQuvabQIbAwYLCQgHAwIDFQIDAxYCAQIeAQIXgAAKCRDImeqTRBlV +WRzJCACbRhx2fYjPGKta69M5dS+kr5UD/CQmsR2t9cB9zyqhratjPnKW9q13+4AG +P3aByT14IH1c5Mha8rJkNYD2wxmC8jrrcPiJIYoRG+W1sUATY/t8wBbNWF+r9h11 +m0lEpsmNVff/jU7SpNN6JQ3P7MHd5V85LlDoXIH6QYCLd0PjKU+jNvjiBe5VX0m9 +a1nacE3xoWc1vbM0DnqEuID78Qgkcrmm0ESeg1h+tRfHxSAyYNc/gPzm8eH6l+hj +gOvUc4Gd6LpBQSF8TcFfT2TZwJh7WVWDvNIP6FWAW7rzmHnX3wwXkGq4REWeVtk5 +yBPp6mOtWDiwaqLJYsoHWU11C8zYnQPEBELr2roBCADrgiWXZMzkQOntZa/NS56+ +CczLFQRQPl/8iJAW1eql/wOJ1UiwGSjT189WCKzE7vtazCIstdCFmwOs4DE6cz4S +UX4HjzjYHZwmMiuSrIefwuZ7cysMBsMXypQFyMSbqwh102xGvmLz3Z++rydx7Fzl +1RC/ny2+FN5dzYPO2DNtNi4dR2tjHktsxBWXAKCmxagAIwyxGouuEqDhYdFtwrA9 +Qy+M5n6fmGa1Dx07WWnbIud4uCilv8LPVKx5aJamDYWM3v7kS8n51MfTzeK/xoRM +2rsgzFdLJqPdbgd2nsD37fngqZnlp7tDxSVSuMckZoSKtq1QsNemtaQSYq7xjPst +AAYp/gcDCNnoEKwFo86JYAsxoD+wQ0zBi5RBM5EphXTpM1qKxmigsKOvBSaMmr0y +VjHtGY3poyV3t6VboOGCsFcaKm0tIdDL7vrxxwyYESETpF29b7QrYcoaLKMG7fsy +t9SUI3UV2H9uUquHgqHtsqz0jYOgm9tYnpesgQ/kOAWI/tej1ZJXUIWEmZMH/W6d +ATNvZ3ivwApfC0qF5G3oPgBSoIuQ/8I+pN/kmuyNAnJWNgagFhA/2VFBvh5XgztV +NW7G//KpR1scsn140SO/wpGBM3Kr4m8ztl9w9U6a7NlQZ2ub3/pIUTpSzyLBxJZ/ +RfuZI7ROdgDMKmEgCYrN2kfp0LIxnYL6ZJu3FDcS4V098lyf5rHvB3PAEdL6Zyhd +qYp3Sx68r0F4vzk5iAIWf6pG2YdfoP2Z48Pmq9xW8qD9iwFcoz9oAzDEMENn6dfq +6MzfoaXEoYp8cR/o+aeEaGUtYBHiaxQcJYx35B9IhsXXA49yRORK8qdwhSHxB3NQ +H3pUWkfw368f/A207hQVs9yYXlEvMZikxl58gldCd3BAPqHm/XzgknRRNQZBPPKJ +BMZebZ22Dm0qDuIqW4GXLB4sLf0+UXydVINIUOlzg+S4jrwx7eZqb6UkRXTIWVo5 +psTsD14wzWBRdUQHZOZD33+M8ugmewvLY/0Uix+2RorkmB7/jqoZvx/MehDwmCZd +VH8sb2wpZ55sj7gCXxvrfieQD/VeH54OwjjbtK56iYq56RVD0h1az8xDY2GZXeT7 +J0c3BGpuoca5xOFWr1SylAr/miEPxOBfnfk8oZQJvZrjSBGjsTbALep2vDJk8ROD +sdQCJuU1RHDrwKHlbUL0NbGRO2juJGsatdWnuVKsFbaFW2pHHkezKuwOcaAJv7Xt +8LRF17czAJ1uaLKwV8Paqx6UIv+089GbWZi7HIkBHwQYAQIACQUCQuvaugIbDAAK +CRDImeqTRBlVWS7XCACDVstKM+SHD6V0bkfO6ampHzj4krKjN0lonN5+7b7WKpgT +QHRYvPY8lUiIrjXGISQqEG9M5Bi5ea1aoBZem0P3U/lKheg0lYtA7dM3BqsA2EfG +RaDD9M5TFCqhy2VFR6Pk0MP7h5bkb2VxLUUQa4oNa1fT3q7zS875NvImO/HZ5UzW +T5d2Z5iwY6I2AOKYKt4kZhzXgbt5j2O3biDDXSfWwwAojWqbqVygepn047KVr7Al +2ug9hkY7tHz7U71HbZasroFgNPmP/UnAxmps4RKM28MRVPTI4cKUIdE3gIKFu3ou +EqEItQ13P+50i3QkALpz8d08tJbceeYzf6I2P4q6 +=QFm5 +-----END PGP PRIVATE KEY BLOCK----- +'); +-- elg1024 / aes128 +insert into encdata (id, data) values (1, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQEOA9k2z2S7c/RmEAQAgVWW0DeLrZ+1thWJGBPp2WRFL9HeNqqWHbKJCXJbz1Uy +faUY7yxVvG5Eutmo+JMiY3mg23/DgVVXHQZsTWpGvGM6djgUNGKUjZDbW6Nog7Mr +e78IywattCOmgUP9vIwwg3OVjuDCN/nVirGQFnXpJBc8DzWqDMWRWDy1M0ZsK7AD +/2JTosSFxUdpON0DKtIY3GLzmh6Nk3iV0g8VgJKUBT1rhCXuMDj3snm//EMm7hTY +PlnObq4mIhgz8NqprmhooxnU0Kapofb3P3wCHPpU14zxhXY8iKO/3JhBq2uFcx4X +uBMwkW4AdNxY/mzJZELteTL8Tr0s7PISk+owb4URpG3n0jsBc0CVULxrjh5Ejkdw +wCM195J6+KbQxOOFQ0b3uOVvv4dEgd/hRERCOq5EPaFhlHegyYJ7YO842vnSDA== +=PABx +-----END PGP MESSAGE----- +'); +-- elg2048 / blowfish +insert into encdata (id, data) values (2, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQIOAywibh/+XMfUEAf+OINhBngEsw4a/IJIeJvUgv1gTQzBwOdQEuc/runr4Oa8 +Skw/Bj0X/zgABVZLem1a35NHaNwaQaCFwMQ41YyWCu+jTdsiyX/Nw0w8LKKz0rNC +vVpG6YuV7Turtsf8a5lXy1K0SHkLlgxQ6c76GS4gtSl5+bsL2+5R1gSRJ9NXqCQP +OHRipEiYwBPqr5R21ZG0FXXNKGOGkj6jt/M/wh3WVtAhYuBI+HPKRfAEjd/Pu/eD +e1zYtkH1dKKFmp44+nF0tTI274xpuso7ShfKYrOK3saFWrl0DWiWteUinjSA1YBY +m7dG7NZ8PW+g1SZWhEoPjEEEHz3kWMvlKheMRDudnQf/dDyX6kZVIAQF/5B012hq +QyVewgTGysowFIDn01uIewoEA9cASw699jw9IoJp+k5WZXnU+INllBLzQxniQCSu +iEcr0x3fYqNtj9QBfbIqyRcY6HTWcmzyOUeGaSyX76j+tRAvtVtXpraFFFnaHB70 +YpXTjLkp8EBafzMghFaKDeXlr2TG/T7rbwcwWrFIwPqEAUKWN5m97Q3eyo8/ioMd +YoFD64J9ovSsgbuU5IpIGAsjxK+NKzg/2STH7zZFEVCtgcIXsTHTZfiwS98/+1H9 +p1DIDaXIcUFV2ztmcKxh9gt2sXRz1W+x6D8O0k3nanU5yGG4miLKaq18fbcA0BD1 ++NIzAfelq6nvvxYKcGcamBMgLo5JkZOBHvyr6RsAKIT5QYc0QTjysTk9l0Am3gYc +G2pAE+3k +=TBHV +-----END PGP MESSAGE----- +'); +-- elg4096 / aes256 +insert into encdata (id, data) values (3, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQQOA7aFBP0Sjh/5EA/+JCgncc8IZmmRjPStWnGf9tVJhgHTn+smIclibGzs0deS +SPSCitzpblwbUDvu964+/5e5Q1l7rRuNN+AgETlEd4eppv7Swn2ChdgOXxRwukcT +Nh3G+PTFvD4ayi7w1db3qvXIt0MwN4Alt436wJmK1oz2Ka9IcyO+wHWrDy1nSGSx +z5x7YEj+EZPgWc/YAvudqE8Jpzd/OT5zSHN09UFkIAk6NxisKaIstbEGFgpqtoDZ +1SJM84XAdL2IcaJ3YY7k/yzwlawhsakKd4GSd5vWmAwvyzzbSiBMfKsDE16ePLNU +ZBF7CzmlCBPZ7YrFAHLpXBXXkCQvzD2BEYOjse50ZEfJ036T7950Ozcdy1EQbGon +nyQ4Gh0PBpnMcBuiXOceWuYzhlzFOzDtlVKdNTxFRDcbEyW2jo9xQYvCCLnYy8EH +2M7S8jCtVYJBbn63a82ELv+3+kWYcsvBJv2ZVBh4ncrBu9o0P+OYS7ApoOU+j6p2 ++t0RXHksqXS1YiUwYF5KSw09EbYMgNZ9G04Px/PxLU6fSC9iDrGX7Xt3kOUP0mku +C518fPckT0zzRXqfFruJNRzDytW50KxkOQZzU1/Az1YlYN9QzWeU4EtLPb2fftZo +D0qH/ln+f9Op5t6sD2fcxZVECU1b/bFtZsxvwH406YL+UQ7hU/XnZrzVVzODal8P +/j1hg7v7BdJqu1DTp9nFWUuwMFcYAczuXn29IG183NZ7Ts4whDeYEhS8eNoLPX4j +txY12ILD/w/3Q4LoW/hPa6OdfEzsn0U5GLf1WiGmJE1H6ft2U/xUnerc/u0kt+FU +WAisArd4MuKtf7B5Vu/VF3kUdrR0hTniUKUivmC4o1jSId31Dufxj4aadVyldXAr +6TNBcdyragZjxEZ6hsBCYzA0Rd1a8atd6OaQoIEEfAzCu5Ks29pydHErStYGjWJ1 +KA5KPLVvjbHpDmRhlCcm8vgpYQsBYEB5gE9fx5yCTlsVhCB6y23h7hfdMqerDqkO +ZOPsO5h+tiHCdIrQ36sMjuINy1/K2rYcXd+Crh2iHcfidpU9fvDz2ihTRNQlhjuT +0cQZM5JhctEx4VXF4LDctRhit7Hn0iqsk604woQfJVvP8O673xSXT/kBY0A/v9C0 +3C4YoFNeSaKwbfZQ/4u1ZFPJxK2IIJa8UGpyAUewLMlzGVVagljybv/f4Z9ERAhy +huq5sMmw8UPsrJF2TUGHz5WSIwoh0J/qovoQI09I9sdEnFczDvRavMO2Mldy3E5i +exz9oewtel6GOmsZQSYWT/vJzbYMmvHNmNpVwwoKrLV6oI3kyQ80GHBwI1WlwHoK +2iRB0w8q4VVvJeYAz8ZIp380cqC3pfO0uZsrOx4g3k4X0jsB5y7rF5xXcZfnVbvG +DYKcOy60/OHMWVvpw6trAoA+iP+cVWPtrbRvLglTVTfYmi1ToZDDipkALBhndQ== +=L/M/ +-----END PGP MESSAGE----- +'); +-- rsaenc2048 / aes128 +insert into encdata (id, data) values (4, ' +-----BEGIN PGP MESSAGE----- +Version: GnuPG v1.4.1 (GNU/Linux) + +hQEMA/0CBsQJt0h1AQf+JyYnCiortj26P11zk28MKOGfWpWyAhuIgwbJXsdQ+e6r +pEyyqs9GC6gI7SNF6+J8B/gsMwvkAL4FHAQCvA4ZZ6eeXR1Of4YG22JQGmpWVWZg +DTyfhA2vkczuqfAD2tgUpMT6sdyGkQ/fnQ0lknlfHgC5GRx7aavOoAKtMqiZW5PR +yae/qR48mjX7Mb+mLvbagv9mHEgQSmHwFpaq2k456BbcZ23bvCmBnCvqV/90Ggfb +VP6gkSoFVsJ19RHsOhW1dk9ehbl51WB3zUOO5FZWwUTY9DJvKblRK/frF0+CXjE4 +HfcZXHSpSjx4haGGTsMvEJ85qFjZpr0eTGOdY5cFhNJAAVP8MZfji7OhPRAoOOIK +eRGOCkao12pvPyFTFnPd5vqmyBbdNpK4Q0hS82ljugMJvM0p3vJZVzW402Kz6iBL +GQ== +=XHkF +-----END PGP MESSAGE----- +'); +-- rsaenc2048 / aes128 (not from gnupg) +insert into encdata (id, data) values (5, ' +-----BEGIN PGP MESSAGE----- + +wcBMA/0CBsQJt0h1AQgAzxZ8j+OTeZ8IlLxfZ/mVd28/gUsCY+xigWBk/anZlK3T +p2tNU2idHzKdAttH2Hu/PWbZp4kwjl9spezYxMqCeBZqtfGED88Y+rqK0n/ul30A +7jjFHaw0XUOqFNlST1v6H2i7UXndnp+kcLfHPhnO5BIYWxB2CYBehItqtrn75eqr +C7trGzU/cr74efcWagbCDSNjiAV7GlEptlzmgVMmNikyI6w0ojEUx8lCLc/OsFz9 +pJUAX8xuwjxDVv+W7xk6c96grQiQlm+FLDYGiGNXoAzx3Wi/howu3uV40dXfY+jx +3WBrhEew5Pkpt1SsWoFnJWOfJ8GLd0ec8vfRCqAIVdLgAeS7NyawQYtd6wuVrEAj +5SMg4Thb4d+g45RksuGLHUUr4qO9tiXglODa4InhmJfgNuLk+RGz4LXjq8wepEmW +vRbgFOG54+Cf4C/gC+HkreDm5JKSKjvvw4B/jC6CDxq+JoziEe2Z1uEjCuEcr+Es +/eGzeOi36BejXPMHeKxXejj5qBBHKV0pHVhZSgffR0TtlXdB967Yl/5agV0R89hI +7Gw52emfnH4Z0Y4V0au2H0k1dR/2IxXdJEWSTG7Be1JHT59p9ei2gSEOrdBMIOjP +tbYYUlmmbvD49bHfThkDiC+oc9947LgQsk3kOOLbNHcjkbrjH8R5kjII4m/SEZA1 +g09T+338SzevBcVXh/cFrQ6/Et+lyyO2LJRUMs69g/HyzJOVWT2Iu8E0eS9MWevY +Qtrkrhrpkl3Y02qEp/j6M03Yu2t6ZF7dp51aJ5VhO2mmmtHaTnCyCc8Fcf72LmD8 +blH2nKZC9d6fi4YzSYMepZpMOFR65M80MCMiDUGnZBB8sEADu2/iVtqDUeG8mAA= +=PHJ1 +-----END PGP MESSAGE----- +'); +-- successful decrypt +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=1 and encdata.id=1; + pgp_pub_decrypt +----------------- + Secret msg +(1 row) + +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=2 and encdata.id=2; +ERROR: Wrong key or corrupt data +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=3 and encdata.id=3; + pgp_pub_decrypt +----------------- + Secret msg +(1 row) + +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=6 and encdata.id=4; + pgp_pub_decrypt +----------------- + Secret message. +(1 row) + +-- wrong key +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=2 and encdata.id=1; +ERROR: Wrong key +-- sign-only key +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=4 and encdata.id=1; +ERROR: No encryption key found +-- rsa: password-protected secret key, wrong password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), '123') +from keytbl, encdata where keytbl.id=7 and encdata.id=4; +ERROR: Wrong key or corrupt data +-- rsa: password-protected secret key, right password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), 'parool') +from keytbl, encdata where keytbl.id=7 and encdata.id=4; + pgp_pub_decrypt +----------------- + Secret message. +(1 row) + +-- password-protected secret key, no password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=5 and encdata.id=1; +ERROR: Need password for secret key +-- password-protected secret key, wrong password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), 'foo') +from keytbl, encdata where keytbl.id=5 and encdata.id=1; +ERROR: Wrong key or corrupt data +-- password-protected secret key, right password +select pgp_pub_decrypt(dearmor(data), dearmor(seckey), 'parool') +from keytbl, encdata where keytbl.id=5 and encdata.id=1; + pgp_pub_decrypt +----------------- + Secret msg +(1 row) + +-- test for a short read from prefix_init +select pgp_pub_decrypt(dearmor(data), dearmor(seckey)) +from keytbl, encdata where keytbl.id=6 and encdata.id=5; +ERROR: Wrong key or corrupt data diff --git a/contrib/pgcrypto/expected/setup_fips.out b/contrib/pgcrypto/expected/setup_fips.out deleted file mode 100644 index 5ee3b794f464..000000000000 --- a/contrib/pgcrypto/expected/setup_fips.out +++ /dev/null @@ -1,7 +0,0 @@ --- Setup for fips.sql test --- We are setting shared_preload_libraries so that we can set the extension GUC --- 'pgcrypto.fips' immediately after creating the extension on master. --- start_ignore -\! gpconfig -c shared_preload_libraries -v '$libdir/pgcrypto' -\! gpstop -air --- end_ignore diff --git a/contrib/pgcrypto/openssl.c b/contrib/pgcrypto/openssl.c index c28cc5b4bdcc..627ef2552847 100644 --- a/contrib/pgcrypto/openssl.c +++ b/contrib/pgcrypto/openssl.c @@ -128,16 +128,24 @@ static unsigned digest_result_size(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; + int result = EVP_MD_CTX_size(digest->ctx); - return EVP_MD_CTX_size(digest->ctx); + if (result < 0) + elog(ERROR, "EVP_MD_CTX_size() failed"); + + return result; } static unsigned digest_block_size(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; + int result = EVP_MD_CTX_block_size(digest->ctx); + + if (result < 0) + elog(ERROR, "EVP_MD_CTX_block_size() failed"); - return EVP_MD_CTX_block_size(digest->ctx); + return result; } static void @@ -145,7 +153,8 @@ digest_reset(PX_MD *h) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestInit_ex(digest->ctx, digest->algo, NULL); + if (!EVP_DigestInit_ex(digest->ctx, digest->algo, NULL)) + elog(ERROR, "EVP_DigestInit_ex() failed"); } static void @@ -153,7 +162,8 @@ digest_update(PX_MD *h, const uint8 *data, unsigned dlen) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestUpdate(digest->ctx, data, dlen); + if (!EVP_DigestUpdate(digest->ctx, data, dlen)) + elog(ERROR, "EVP_DigestUpdate() failed"); } static void @@ -161,7 +171,8 @@ digest_finish(PX_MD *h, uint8 *dst) { OSSLDigest *digest = (OSSLDigest *) h->p.ptr; - EVP_DigestFinal_ex(digest->ctx, dst, NULL); + if (!EVP_DigestFinal_ex(digest->ctx, dst, NULL)) + elog(ERROR, "EVP_DigestFinal_ex() failed"); } static void @@ -381,6 +392,8 @@ gen_ossl_decrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, { if (!EVP_DecryptInit_ex(od->evp_ctx, od->evp_ciph, NULL, NULL, NULL)) return PXE_CIPHER_INIT; + if (!EVP_CIPHER_CTX_set_padding(od->evp_ctx, 0)) + return PXE_CIPHER_INIT; if (!EVP_CIPHER_CTX_set_key_length(od->evp_ctx, od->klen)) return PXE_CIPHER_INIT; if (!EVP_DecryptInit_ex(od->evp_ctx, NULL, NULL, od->key, od->iv)) @@ -405,6 +418,8 @@ gen_ossl_encrypt(PX_Cipher *c, const uint8 *data, unsigned dlen, { if (!EVP_EncryptInit_ex(od->evp_ctx, od->evp_ciph, NULL, NULL, NULL)) return PXE_CIPHER_INIT; + if (!EVP_CIPHER_CTX_set_padding(od->evp_ctx, 0)) + return PXE_CIPHER_INIT; if (!EVP_CIPHER_CTX_set_key_length(od->evp_ctx, od->klen)) return PXE_CIPHER_INIT; if (!EVP_EncryptInit_ex(od->evp_ctx, NULL, NULL, od->key, od->iv)) diff --git a/contrib/pgcrypto/px.c b/contrib/pgcrypto/px.c index 0f02fb56c4fb..2c6704e25777 100644 --- a/contrib/pgcrypto/px.c +++ b/contrib/pgcrypto/px.c @@ -292,6 +292,7 @@ static int combo_decrypt(PX_Combo *cx, const uint8 *data, unsigned dlen, uint8 *res, unsigned *rlen) { + int err = 0; unsigned bs, i, pad; @@ -317,7 +318,9 @@ combo_decrypt(PX_Combo *cx, const uint8 *data, unsigned dlen, /* decrypt */ *rlen = dlen; - px_cipher_decrypt(c, data, dlen, res); + err = px_cipher_decrypt(c, data, dlen, res); + if (err) + return err; /* unpad */ if (bs > 1 && cx->padding) diff --git a/contrib/pgcrypto/sql/fips.sql b/contrib/pgcrypto/sql/fips.sql deleted file mode 100644 index d0b0d71d0755..000000000000 --- a/contrib/pgcrypto/sql/fips.sql +++ /dev/null @@ -1,40 +0,0 @@ -ALTER DATABASE contrib_regression SET pgcrypto.fips TO on; -\c contrib_regression - -SHOW pgcrypto.fips; - -CREATE TABLE fipstest (data text, res text, salt text); -INSERT INTO fipstest VALUES ('password', '', ''); - -SELECT 'Test digest md5: EXPECTED ERROR FAIL FIPS' as comment; -SELECT digest('santa claus', 'md5'); - -SELECT 'Test digest sha256: EXPECTED PASS' as comment; -SELECT digest('santa claus', 'sha256'); - -SELECT 'Test hmac md5: EXPECTED ERROR FAIL FIPS' as comment; -SELECT hmac('santa claus', 'aaa', 'md5'); - -SELECT 'Test hmac sha256: EXPECTED PASS' as comment; -SELECT hmac('santa claus', 'aaa', 'sha256'); - -SELECT 'Test gen_salt : EXPECTED FAIL FIPS' as comment; -UPDATE fipstest SET salt = gen_salt('md5'); - -SELECT 'Test crypt : EXPECTED FAIL FIPS' as comment; -UPDATE fipstest SET res = crypt(data, salt); -SELECT res = crypt(data, res) AS "worked" FROM fipstest; - -SELECT 'Test pgp : EXPECTED PASS' as comment; -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=aes256'), 'mypass'); - -SELECT 'Test pgp : EXPECTED FAIL FIPS' as comment; -select pgp_sym_decrypt(pgp_sym_encrypt('santa clause', 'mypass', 'cipher-algo=bf'), 'mypass'); - -SELECT 'Test raw encrypt : EXPECTED PASS' as comment; -SELECT encrypt('santa claus', 'mypass', 'aes') as raw_aes; - -SELECT 'Test raw encrypt : EXPECTED FAIL FIPS' as comment; -SELECT encrypt('santa claus', 'mypass', 'bf') as raw_blowfish; - -DROP TABLE fipstest; diff --git a/gpAux/Makefile b/gpAux/Makefile index 410965e007c3..cb915e37f022 100644 --- a/gpAux/Makefile +++ b/gpAux/Makefile @@ -120,13 +120,14 @@ DEFPORT=5432 ORCA_CONFIG=--enable-orca -rhel6_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -rhel7_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -rhel8_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -rocky8_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -ol8_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -ubuntu18.04_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm -sles12_x86_64_CONFIGFLAGS=--with-quicklz --with-gssapi --enable-mapreduce --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-pythonsrc-ext --with-uuid=e2fs --with-llvm +rhel6_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rhel7_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rhel8_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rocky8_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +rocky9_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +ol8_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +ubuntu18.04_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm +sles12_x86_64_CONFIGFLAGS=--with-gssapi --enable-orafce --enable-ic-proxy ${ORCA_CONFIG} --enable-gpcloud --with-libxml --with-openssl --with-pam --with-ldap --with-uuid=e2fs --with-llvm BLD_CONFIGFLAGS=$($(BLD_ARCH)_CONFIGFLAGS) CONFIGFLAGS=$(strip $(BLD_CONFIGFLAGS) --with-pgport=$(DEFPORT) $(BLD_DEPLOYMENT_SETTING)) diff --git a/gpAux/gpdemo/generate_certs.sh b/gpAux/gpdemo/generate_certs.sh index b2f628d8455c..1aa9864a963a 100755 --- a/gpAux/gpdemo/generate_certs.sh +++ b/gpAux/gpdemo/generate_certs.sh @@ -11,7 +11,7 @@ California Palo Alto Pivotal GPDB -127.0.0.1 +localhost gpdb@127.0.0.1 . . @@ -29,10 +29,10 @@ chmod og-rwx ${key} mkdir -p certificate/gpfdists -cp server.key certificate/gpfdists/server.key -cp server.crt certificate/gpfdists/server.crt -cp server.key certificate/gpfdists/client.key -cp server.crt certificate/gpfdists/client.crt -cp server.crt certificate/gpfdists/root.crt +cp $key certificate/gpfdists/server.key +cp $cert certificate/gpfdists/server.crt +cp $key certificate/gpfdists/client.key +cp $cert certificate/gpfdists/client.crt +cp $cert certificate/gpfdists/root.crt rm -f server.* privkey.pem diff --git a/gpMgmt/Makefile.behave b/gpMgmt/Makefile.behave index b0b7b83346d1..759064966064 100644 --- a/gpMgmt/Makefile.behave +++ b/gpMgmt/Makefile.behave @@ -1,4 +1,5 @@ PEXPECT_LIB=$(GPHOME)/bin/lib +TEST_DIR=$(CURDIR)/test # To pass in custom flags as behave args(such as multiple flags), # use flags=--tags=foo, --tags=-bar @@ -6,9 +7,9 @@ behave: @which behave || (echo "behave not found. Run pip install -r gpMgmt/requirements-dev.txt" && exit 1) @echo "Running behave on management scripts..." @if [ -n """$(flags)""" ]; then \ - PYTHONPATH=$$PYTHONPATH:$(PEXPECT_LIB) behave $(CURDIR)/test/behave/* -s -k $(flags) 2>&1 ; \ + PYTHONPATH=$$PYTHONPATH:$(PEXPECT_LIB):$(TEST_DIR) behave $(CURDIR)/test/behave/* -s -k $(flags) 2>&1 ; \ elif [ -n """$(tags)""" ]; then \ - PYTHONPATH=$$PYTHONPATH:$(PEXPECT_LIB) behave $(CURDIR)/test/behave/* -s -k --tags=$(tags) 2>&1 ; \ + PYTHONPATH=$$PYTHONPATH:$(PEXPECT_LIB):$(TEST_DIR) behave $(CURDIR)/test/behave/* -s -k --tags=$(tags) 2>&1 ; \ else \ echo "Please specify tags=tagname or flags=[behave flags]"; \ exit 1; \ diff --git a/gpMgmt/bin/analyzedb b/gpMgmt/bin/analyzedb index a2ad49a7ff30..2a0a0ce81d73 100755 --- a/gpMgmt/bin/analyzedb +++ b/gpMgmt/bin/analyzedb @@ -25,16 +25,15 @@ from contextlib import closing import pipes # for shell-quoting, pipes.quote() import fcntl import itertools - +import psycopg2 try: - import pg - from gppylib import gplog, pgconf, userinput from gppylib.commands.base import Command, WorkerPool, Worker from gppylib.operations import Operation from gppylib.gpversion import GpVersion from gppylib.db import dbconn from gppylib.operations.unix import CheckDir, CheckFile, MakeDir + from gppylib.utils import escape_string except ImportError as e: sys.exit('Cannot import modules. Please check that you have sourced greenplum_path.sh. Detail: ' + str(e)) @@ -166,7 +165,7 @@ def validate_schema_exists(pg_port, dbname, schema): try: dburl = dbconn.DbURL(port=pg_port, dbname=dbname) conn = dbconn.connect(dburl) - count = dbconn.querySingleton(conn, "select count(*) from pg_namespace where nspname='%s';" % pg.escape_string(schema)) + count = dbconn.querySingleton(conn, "select count(*) from pg_namespace where nspname='%s';" % escape_string(schema)) if count == 0: raise ExceptionNoStackTraceNeeded("Schema %s does not exist in database %s." % (schema, dbname)) finally: @@ -213,7 +212,7 @@ def get_partition_state_tuples(pg_port, dbname, catalog_schema, partition_info): try: modcount_sql = "select to_char(coalesce(sum(modcount::bigint), 0), '999999999999999999999') from gp_dist_random('%s.%s')" % (catalog_schema, tupletable) modcount = dbconn.querySingleton(conn, modcount_sql) - except pg.DatabaseError as e: + except psycopg2.DatabaseError as e: if "does not exist" in str(e): logger.info("Table %s.%s (%s) no longer exists and will not be analyzed", schemaname, partition_name, tupletable) else: @@ -971,7 +970,7 @@ def get_oid_str(table_list): def regclass_schema_tbl(schema, tbl): schema_tbl = "%s.%s" % (escape_identifier(schema), escape_identifier(tbl)) - return "to_regclass('%s')" % (pg.escape_string(schema_tbl)) + return "to_regclass('%s')" % (escape_string(schema_tbl)) # Escape double-quotes in a string, so that the resulting string is suitable for @@ -1239,7 +1238,7 @@ def validate_tables(conn, tablenames): while curr_batch < nbatches: batch = tablenames[curr_batch * batch_size:(curr_batch + 1) * batch_size] - oid_str = ','.join(map((lambda x: "('%s')" % pg.escape_string(x)), batch)) + oid_str = ','.join(map((lambda x: "('%s')" % escape_string(x)), batch)) if not oid_str: break @@ -1255,7 +1254,7 @@ def get_include_cols_from_exclude(conn, schema, table, exclude_cols): """ Given a list of excluded columns of a table, get the list of included columns """ - quoted_exclude_cols = ','.join(["'%s'" % pg.escape_string(x) for x in exclude_cols]) + quoted_exclude_cols = ','.join(["'%s'" % escape_string(x) for x in exclude_cols]) oid_str = regclass_schema_tbl(schema, table) cols = run_sql(conn, GET_INCLUDED_COLUMNS_FROM_EXCLUDE_SQL % (oid_str, quoted_exclude_cols)) @@ -1271,7 +1270,7 @@ def validate_columns(conn, schema, table, column_list): return sql = VALIDATE_COLUMN_NAMES_SQL % (regclass_schema_tbl(schema, table), - ','.join(["'%s'" % pg.escape_string(x) for x in column_list])) + ','.join(["'%s'" % escape_string(x) for x in column_list])) valid_col_count = dbconn.querySingleton(conn, sql) if int(valid_col_count) != len(column_list): diff --git a/gpMgmt/bin/gpactivatestandby b/gpMgmt/bin/gpactivatestandby index a38ec8cb00ac..f65b9bdcc825 100755 --- a/gpMgmt/bin/gpactivatestandby +++ b/gpMgmt/bin/gpactivatestandby @@ -21,10 +21,9 @@ import time import shutil import tempfile from datetime import datetime, timedelta - +import psycopg2 # import GPDB modules try: - import pg as pygresql from gppylib.commands import unix, gp, pg from gppylib.db import dbconn from gppylib.gpparseopts import OptParser, OptChecker, OptionGroup, SUPPRESS_HELP @@ -341,7 +340,7 @@ def promote_standby(coordinator_data_dir): dbconn.execSQL(conn, 'CHECKPOINT') conn.close() return True - except pygresql.InternalError as e: + except (psycopg2.InternalError, psycopg2.OperationalError) as e: pass time.sleep(1) diff --git a/gpMgmt/bin/gpcheckcat b/gpMgmt/bin/gpcheckcat index 1b047bafa0b0..938ff57633a9 100755 --- a/gpMgmt/bin/gpcheckcat +++ b/gpMgmt/bin/gpcheckcat @@ -27,23 +27,20 @@ import re import sys import time from functools import reduce - +import psycopg2 +from psycopg2 import extras +from contextlib import closing try: from gppylib import gplog - from gppylib.db import dbconn from gppylib.gpcatalog import * from gppylib.commands.unix import * from gppylib.commands.gp import conflict_with_gpexpand from gppylib.system.info import * - from pgdb import DatabaseError from gpcheckcat_modules.unique_index_violation_check import UniqueIndexViolationCheck from gpcheckcat_modules.leaked_schema_dropper import LeakedSchemaDropper from gpcheckcat_modules.repair import Repair from gpcheckcat_modules.foreign_key_check import ForeignKeyCheck from gpcheckcat_modules.orphaned_toast_tables_check import OrphanedToastTablesCheck - - import pg - except ImportError as e: sys.exit('Error: unable to import module: ' + str(e)) @@ -136,7 +133,7 @@ class Global(): self.dbname = None self.firstdb = None self.alldb = [] - self.db = {} + self.conn = {} self.tmpdir = None self.reset_stmt_queues() @@ -205,30 +202,27 @@ def usage(exitarg=None): ############################### def getversion(): - db = connect() - curs = db.query(''' - select regexp_replace(version(), - E'.*PostgreSQL [^ ]+ .Greenplum Database ([1-9]+.[0-9]+|main).*', - E'\\\\1') as ver;''') - - row = curs.getresult()[0] - version = row[0] - - logger.debug('got version %s' % version) - return version - + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute(''' + select regexp_replace(version(), + E'.*PostgreSQL [^ ]+ .Greenplum Database ([1-9]+.[0-9]+|main).*', + E'\\\\1') as ver;''') + row = curs.fetchone() + version = row[0] + logger.debug('got version %s' % version) + return version ############################### def getalldbs(): """ get all connectable databases """ - db = connect() - curs = db.query(''' - select datname from pg_database where datallowconn order by datname ''') - row = curs.getresult() - return row - + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute('''select datname from pg_database where datallowconn order by datname''') + row = curs.fetchall() + return row ############################### def parseCommandLine(): @@ -337,19 +331,21 @@ def connect(user=None, password=None, host=None, port=None, try: logger.debug('connecting to %s:%s %s' % (host, port, database)) - db = pg.connect(host=host, port=port, user=user, - passwd=password, dbname=database, opt=options) - - except pg.InternalError as ex: + conn = psycopg2.connect(host=host, port=port, user=user, + password=password, dbname=database, options=options) + ## Don't execute query in a transaction block. + conn.set_session(autocommit=True) + except (psycopg2.InternalError, psycopg2.OperationalError) as ex: logger.fatal('could not connect to %s: "%s"' % (database, str(ex).strip())) exit(1) logger.debug('connected with %s:%s %s' % (host, port, database)) - return db + return conn -############# +# NOTE: We cannot use connect2() with contextmanager, since we manage the connection +# ourselves. def connect2(cfgrec, user=None, password=None, database=None, utilityMode=True): host = cfgrec['address'] port = cfgrec['port'] @@ -361,22 +357,22 @@ def connect2(cfgrec, user=None, password=None, database=None, utilityMode=True): key = "%s.%s.%s.%s.%s.%s.%s" % (host, port, datadir, user, password, database, str(utilityMode)) - conns = GV.db.get(key) + conns = GV.conn.get(key) if conns: return conns[0] conn = connect(host=host, port=port, user=user, password=password, database=database, utilityMode=utilityMode) if conn: - GV.db[key] = [conn, cfgrec] + GV.conn[key] = [conn, cfgrec] return conn class execThread(Thread): - def __init__(self, cfg, db, qry): + def __init__(self, cfg, conn, qry): self.cfg = cfg - self.db = db + self.conn = conn self.qry = qry self.curs = None self.error = None @@ -384,11 +380,11 @@ class execThread(Thread): def run(self): try: - self.curs = self.db.query(self.qry) + self.curs = self.conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + self.curs.execute(self.qry) except BaseException as e: self.error = e - def processThread(threads): batch = [] for th in threads: @@ -417,9 +413,9 @@ def connect2run(qry, col=None): # parallelise queries for dbid in GV.cfg: c = GV.cfg[dbid] - db = connect2(c) + conn = connect2(c) - thread = execThread(c, db, qry) + thread = execThread(c, conn, qry) thread.start() logger.debug('launching query thread %s for dbid %i' % (thread.name, dbid)) @@ -439,8 +435,8 @@ def connect2run(qry, col=None): err = [] for [cfg, curs] in batch: if col is None: - col = curs.listfields() - for row in curs.dictresult(): + col = [desc[0] for desc in curs.description] + for row in curs.fetchall(): err.append([cfg, col, row]) return err @@ -458,7 +454,6 @@ def formatErr(c, col, row): ############# def getGPConfiguration(): cfg = {} - db = connect() # note that in 4.0, sql commands cannot be run against the segment mirrors directly # so we filter out non-primary segment databases in the query qry = ''' @@ -468,13 +463,14 @@ def getGPConfiguration(): FROM gp_segment_configuration WHERE (role = 'p' or content < 0 ) ''' - curs = db.query(qry) - for row in curs.dictresult(): - if row['content'] == -1 and not row['isprimary']: - continue # skip standby coordinator - cfg[row['dbid']] = row - db.close() - return cfg + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + for row in curs.fetchall(): + if row['content'] == -1 and not row['isprimary']: + continue # skip standby coordinator + cfg[row['dbid']] = row + return cfg def checkDistribPolicy(): logger.info('-----------------------------------') @@ -488,27 +484,27 @@ def checkDistribPolicy(): where pk.contype in('p', 'u') and d.policytype = 'p' and d.distkey = '' ''' - db = connect2(GV.cfg[GV.coordinator_dbid]) try: - curs = db.query(qry) - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) - - if not err: - logger.info('[OK] randomly distributed tables') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] randomly distributed tables') - logger.error('pg_constraint has %d issue(s)' % len(err)) - logger.error(qry) - for e in err: - logger.error(formatErr(e[0], e[1], e[2])) - for e in err: - cons = e[2] - removeIndexConstraint(cons['nspname'], cons['relname'], - cons['constraint']) + conn = connect2(GV.cfg[GV.coordinator_dbid]) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) + + if not err: + logger.info('[OK] randomly distributed tables') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] randomly distributed tables') + logger.error('pg_constraint has %d issue(s)' % len(err)) + logger.error(qry) + for e in err: + logger.error(formatErr(e[0], e[1], e[2])) + cons = e[2] + removeIndexConstraint(cons['nspname'], cons['relname'], + cons['constraint']) except Exception as e: setError(ERROR_NOREPAIR) myprint('[ERROR] executing test: checkDistribPolicy') @@ -530,22 +526,23 @@ def checkDistribPolicy(): and not d.distkey::int2[] operator(pg_catalog.<@) pk.conkey ''' try: - curs = db.query(qry) - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) - - if not err: - logger.info('[OK] unique constraints') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] unique constraints') - logger.error('pg_constraint has %d issue(s)' % len(err)) - logger.error(qry) - for e in err: logger.error(formatErr(e[0], e[1], e[2])) + conn = connect2(GV.cfg[GV.coordinator_dbid]) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + err = [] + curs.execute(qry) + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], ('nspname', 'relname', 'constraint'), row]) + + if not err: + logger.info('[OK] unique constraints') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] unique constraints') + logger.error('pg_constraint has %d issue(s)' % len(err)) + logger.error(qry) for e in err: + logger.error(formatErr(e[0], e[1], e[2])) cons = e[2] removeIndexConstraint(cons['nspname'], cons['relname'], cons['constraint']) @@ -561,7 +558,6 @@ def checkPartitionIntegrity(): logger.info('-----------------------------------') logger.info('Checking pg_partition ...') err = [] - db = connect() # Check for the numsegments value of parent and child partition from the gp_distribution_policy table qry = ''' @@ -573,53 +569,55 @@ def checkPartitionIntegrity(): and not (inhrelid in (select ftrelid from pg_catalog.pg_foreign_table) and child.numsegments = NULL); ''' try: - curs = db.query(qry) - cols = ('inhparent', 'inhrelid', 'numsegments_parent', 'numsegments_child') - col_names = { - 'inhparent': 'table', - 'inhrelid': 'affected child', - 'numsegments_parent': 'parent numsegments value', - 'numsegments_child': 'child numsegments value', - } - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], cols, row]) - - if not err: - logger.info('[OK] partition numsegments check') - else: - err_count = len(err) - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] partition numsegments check') - logger.error('partition numsegments check found %d issue(s)' % err_count) - if err_count > 100: - logger.error(qry) + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + cols = ('inhparent', 'inhrelid', 'numsegments_parent', 'numsegments_child') + col_names = { + 'inhparent': 'table', + 'inhrelid': 'affected child', + 'numsegments_parent': 'parent numsegments value', + 'numsegments_child': 'child numsegments value', + } + + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], cols, row]) + + if not err: + logger.info('[OK] partition numsegments check') + else: + err_count = len(err) + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] partition numsegments check') + logger.error('partition numsegments check found %d issue(s)' % err_count) + if err_count > 100: + logger.error(qry) - myprint( - '[ERROR]: child partition(s) have different numsegments value ' - 'from the root partition. Check the gpcheckcat log for details.' - ) - logger.error('The following tables have different numsegments value (showing at most 100 rows):') + myprint( + '[ERROR]: child partition(s) have different numsegments value ' + 'from the root partition. Check the gpcheckcat log for details.' + ) + logger.error('The following tables have different numsegments value (showing at most 100 rows):') - # report at most 100 rows, for brevity - err = err[:100] + # report at most 100 rows, for brevity + err = err[:100] - for index, e in enumerate(err): - cfg = e[0] - col = e[1] - row = e[2] + for index, e in enumerate(err): + cfg = e[0] + col = e[1] + row = e[2] - if index == 0: - logger.error("--------") - logger.error(" " + " | ".join(map(col_names.get, col))) - logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) + if index == 0: + logger.error("--------") + logger.error(" " + " | ".join(map(col_names.get, col))) + logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) - logger.error(" " + " | ".join([str(row[x]) for x in col])) + logger.error(" " + " | ".join([str(row[x]) for x in col])) - if err_count > 100: - logger.error(" ...") + if err_count > 100: + logger.error(" ...") except Exception as e: setError(ERROR_NOREPAIR) @@ -642,74 +640,74 @@ def checkPartitionIntegrity(): and (select isleaf from pg_partition_tree(inhparent) where relid = inhrelid)); ''' try: - curs = db.query(qry) - cols = ('inhparent', 'inhrelid', 'dby_parent', 'dby_child') - col_names = { - 'inhparent': 'table', - 'inhrelid': 'affected child', - 'dby_parent': 'table distribution key', - 'dby_child': 'child distribution key', - } - - err = [] - for row in curs.dictresult(): - err.append([GV.cfg[GV.coordinator_dbid], cols, row]) - - if not err: - logger.info('[OK] partition distribution policy check') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] partition distribution policy check') - logger.error('partition distribution policy check found %d issue(s)' % len(err)) - if len(err) > 100: - logger.error(qry) - - myprint( - '[ERROR]: child partition(s) are distributed differently from ' - 'the root partition, and must be manually redistributed, for ' - 'some tables. Check the gpcheckcat log for details.' - ) - logger.error('The following tables must be manually redistributed:') - - count = 0 - for e in err: - cfg = e[0] - col = e[1] - row = e[2] - - # TODO: generate a repair script for this row. This is - # difficult, since we can't redistribute child partitions - # directly. - - # report at most 100 rows, for brevity - if count == 100: - logger.error("...") - count += 1 - if count > 100: - continue - - if count == 0: - logger.error("--------") - logger.error(" " + " | ".join(map(col_names.get, col))) - logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) - - logger.error(" " + " | ".join([str(row[x]) for x in col])) - count += 1 - - logger.error( - 'Execute an ALTER TABLE ... SET DISTRIBUTED BY statement, with ' - 'the desired distribution key, on the partition root for each ' - 'affected table.' - ) + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + cols = ('inhparent', 'inhrelid', 'dby_parent', 'dby_child') + col_names = { + 'inhparent': 'table', + 'inhrelid': 'affected child', + 'dby_parent': 'table distribution key', + 'dby_child': 'child distribution key', + } + + err = [] + for row in curs.fetchall(): + err.append([GV.cfg[GV.coordinator_dbid], cols, row]) + + if not err: + logger.info('[OK] partition distribution policy check') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] partition distribution policy check') + logger.error('partition distribution policy check found %d issue(s)' % len(err)) + if len(err) > 100: + logger.error(qry) + + myprint( + '[ERROR]: child partition(s) are distributed differently from ' + 'the root partition, and must be manually redistributed, for ' + 'some tables. Check the gpcheckcat log for details.' + ) + logger.error('The following tables must be manually redistributed:') + + count = 0 + for e in err: + cfg = e[0] + col = e[1] + row = e[2] + + # TODO: generate a repair script for this row. This is + # difficult, since we can't redistribute child partitions + # directly. + + # report at most 100 rows, for brevity + if count == 100: + logger.error("...") + count += 1 + if count > 100: + continue + + if count == 0: + logger.error("--------") + logger.error(" " + " | ".join(map(col_names.get, col))) + logger.error(" " + "-+-".join(['-' * len(col_names[x]) for x in col])) + + logger.error(" " + " | ".join([str(row[x]) for x in col])) + count += 1 + + logger.error( + 'Execute an ALTER TABLE ... SET DISTRIBUTED BY statement, with ' + 'the desired distribution key, on the partition root for each ' + 'affected table.' + ) except Exception as e: setError(ERROR_NOREPAIR) myprint('[ERROR] executing test: checkPartitionIntegrity') myprint(' Execution error: ' + str(e)) - db.close() - checkPoliciesRepair() ############# @@ -776,7 +774,7 @@ Produce repair scripts to remove dangling entries of gp_fastsequence: ''' -def removeFastSequence(db): +def removeFastSequence(conn): ''' MPP-14758: gp_fastsequence does not get cleanup after a failed transaction (AO/CO) Note: this is slightly different from the normal foreign key check @@ -802,14 +800,15 @@ def removeFastSequence(db): ON r.gp_segment_id = cfg.content WHERE cfg.role = 'p'; """ - curs = db.query(qry) - for row in curs.dictresult(): - seg = row['dbid'] # dbid of targeted segment - name = 'gp_fastsequence tuple' # for comment purposes - table = 'gp_fastsequence' # table name - cols = {'objid': row['objid']} # column name and value - objname = 'gp_fastsequence' # for comment purposes - buildRemove(seg, name, table, cols, objname) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + for row in curs.fetchall(): + seg = row['dbid'] # dbid of targeted segment + name = 'gp_fastsequence tuple' # for comment purposes + table = 'gp_fastsequence' # table name + cols = {'objid': row['objid']} # column name and value + objname = 'gp_fastsequence' # for comment purposes + buildRemove(seg, name, table, cols, objname) except Exception as e: logger.error('removeFastSequence: ' + str(e)) @@ -871,42 +870,41 @@ def drop_leaked_schemas(leaked_schema_dropper, dbname): logger.info('-----------------------------------') logger.info('Checking for leaked temporary schemas') - db_connection = connect(database=dbname) try: - dropped_schemas = leaked_schema_dropper.drop_leaked_schemas(db_connection) - if not dropped_schemas: - logger.info('[OK] temporary schemas') - else: - logger.info('[FAIL] temporary schemas') - myprint("Found and dropped %d unbound temporary schemas" % len(dropped_schemas)) - logger.error('Dropped leaked schemas \'%s\' in the database \'%s\'' % (dropped_schemas, dbname)) + with closing(connect(database=dbname)) as conn: + dropped_schemas = leaked_schema_dropper.drop_leaked_schemas(conn) + if not dropped_schemas: + logger.info('[OK] temporary schemas') + else: + logger.info('[FAIL] temporary schemas') + myprint("Found and dropped %d unbound temporary schemas" % len(dropped_schemas)) + logger.error('Dropped leaked schemas \'%s\' in the database \'%s\'' % (dropped_schemas, dbname)) except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) - finally: - db_connection.close() def checkDepend(): # Check for dependencies on non-existent objects logger.info('-----------------------------------') logger.info('Checking Object Dependencies') - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: - # Catalogs that link up to pg_depend/pg_shdepend - qry = """ - select relname from pg_class c - where relkind='r' - and relnamespace=%d - and exists (select 1 from pg_attribute a where attname = 'oid' and a.attrelid = c.oid) - """ % PG_CATALOG_OID - curs = db.query(qry) - catalogs = [] - for row in curs.getresult(): - catalogs.append(row[0]) - - checkDependJoinCatalog(catalogs) - checkCatalogJoinDepend(catalogs) + # Catalogs that link up to pg_depend/pg_shdepend + qry = """ + select relname from pg_class c + where relkind='r' + and relnamespace=%d + and exists (select 1 from pg_attribute a where attname = 'oid' and a.attrelid = c.oid) + """ % PG_CATALOG_OID + curs.execute(qry) + catalogs = [] + for row in curs.fetchall(): + catalogs.append(row[0]) + + checkDependJoinCatalog(catalogs) + checkCatalogJoinDepend(catalogs) def checkDependJoinCatalog(catalogs): # Construct subquery that will verify that all (classid, objid) @@ -1063,7 +1061,6 @@ def checkOwners(): # # - Between 3.3 and 4.0 the ao segment columns migrated from pg_class # to pg_appendonly. - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) qry = ''' select distinct n.nspname, coalesce(o.relname, c.relname) as relname, a.rolname, m.rolname as coordinator_rolname @@ -1080,20 +1077,21 @@ def checkOwners(): where c.relowner <> r.relowner ''' try: - curs = db.query(qry) - - rows = [] - for row in curs.dictresult(): - rows.append(row) - - if len(rows) == 0: - logger.info('[OK] table ownership') - else: - GV.checkStatus = False - setError(ERROR_REMOVE) - logger.info('[FAIL] table ownership') - logger.error('found %d table ownership issue(s)' % len(rows)) - logger.error('%s' % qry) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + rows = [] + for row in curs.fetchall(): + rows.append(row) + + if len(rows) == 0: + logger.info('[OK] table ownership') + else: + GV.checkStatus = False + setError(ERROR_REMOVE) + logger.info('[FAIL] table ownership') + logger.error('found %d table ownership issue(s)' % len(rows)) + logger.error('%s' % qry) for row in rows[0:100]: logger.error(' %s.%s relowner %s != %s' % (row['nspname'], row['relname'], row['rolname'], @@ -1117,7 +1115,6 @@ def checkOwners(): # - Ignore implementation types of pg_class entries - they should be # in the check above since ALTER TABLE is required to fix them, not # ALTER TYPE. - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) qry = ''' select distinct n.nspname, t.typname, a.rolname, m.rolname as coordinator_rolname from gp_dist_random('pg_type') r @@ -1128,27 +1125,28 @@ def checkOwners(): where r.typowner <> t.typowner ''' try: - curs = db.query(qry) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) - rows = [] - for row in curs.dictresult(): - rows.append(row) + rows = [] + for row in curs.fetchall(): + rows.append(row) - if len(rows) == 0: - logger.info('[OK] type ownership') - else: - GV.checkStatus = False - setError(ERROR_NOREPAIR) - logger.info('[FAIL] type ownership') - logger.error('found %d type ownership issue(s)' % len(rows)) - logger.error('%s' % qry) + if len(rows) == 0: + logger.info('[OK] type ownership') + else: + GV.checkStatus = False + setError(ERROR_NOREPAIR) + logger.info('[FAIL] type ownership') + logger.error('found %d type ownership issue(s)' % len(rows)) + logger.error('%s' % qry) for row in rows[0:100]: logger.error(' %s.%s typeowner %s != %s' % (row['nspname'], row['typname'], row['rolname'], row['coordinator_rolname'])) if len(rows) > 100: logger.error("...") - except Exception as e: setError(ERROR_NOREPAIR) myprint("[ERROR] executing test: check type ownership") @@ -1172,15 +1170,14 @@ def checkOwners(): def closeDbs(): - for key, conns in GV.db.items(): - db = conns[0] - db.close() - GV.db = {} # remove everything + for key, conns in GV.conn.items(): + conn = conns[0] + conn.close() + GV.conn = {} # remove everything # ------------------------------------------------------------------------------- def getCatObj(namestr): - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) try: cat = GV.catalog.getCatalogTable(namestr) except Exception as e: @@ -1250,25 +1247,25 @@ def checkTableACL(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() - - if nrows == 0: - logger.info('[OK] Cross consistency acl check for ' + catname) - else: - GV.checkStatus = False - setError(ERROR_NOREPAIR) - GV.aclStatus = False - logger.info('[FAIL] Cross consistency acl check for ' + catname) - logger.error(' %s acl check has %d issue(s)' % (catname, nrows)) - - fields = curs.listfields() - gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): - gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) - processACLResult(catname, fields, curs.getresult()) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: + curs = db.execute(qry) + nrows = curs.rowcount + if nrows == 0: + logger.info('[OK] Cross consistency acl check for ' + catname) + else: + GV.checkStatus = False + setError(ERROR_NOREPAIR) + GV.aclStatus = False + logger.info('[FAIL] Cross consistency acl check for ' + catname) + logger.error(' %s acl check has %d issue(s)' % (catname, nrows)) + + fields = [desc[0] for desc in curs.description] + gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) + for row in curs.getresult(): + gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) + processACLResult(catname, fields, curs.getresult()) except Exception as e: setError(ERROR_NOREPAIR) GV.aclStatus = False @@ -1289,9 +1286,9 @@ def checkForeignKey(cat_tables=None): if not cat_tables: cat_tables = GV.catalog.getCatalogTables() - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) try: - foreign_key_check = ForeignKeyCheck(db_connection, logger, GV.opt['-S'], autoCast) + foreign_key_check = ForeignKeyCheck(conn, logger, GV.opt['-S'], autoCast) foreign_key_issues = foreign_key_check.runCheck(cat_tables) if foreign_key_issues: GV.checkStatus = False @@ -1301,13 +1298,14 @@ def checkForeignKey(cat_tables=None): processForeignKeyResult(catname, pkcatname, fields, results) if catname == 'gp_fastsequence' and pkcatname == 'pg_class': setError(ERROR_REMOVE) - removeFastSequence(db_connection) + removeFastSequence(conn) else: setError(ERROR_NOREPAIR) except Exception as ex: setError(ERROR_NOREPAIR) GV.foreignKeyStatus = False myprint(' Execution error: ' + str(ex)) + # ------------------------------------------------------------------------------- @@ -1377,40 +1375,39 @@ def checkTableMissingEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() - results = curs.getresult() - fields = curs.listfields() - - if nrows != 0: - results = filterSpuriousFailures(catname, fields, results) - nrows = len(results) - - if nrows == 0: - logger.info('[OK] Checking for missing or extraneous entries for ' + catname) - else: - if catname in ['pg_constraint']: - logger_with_level = logger.warning - log_level = logging.WARNING + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + with conn.cursor() as curs: + curs.execute(qry) + nrows = curs.rowcount + results = curs.fetchall() + fields = [desc[0] for desc in curs.description] + + if nrows != 0: + results = filterSpuriousFailures(catname, fields, results) + nrows = len(results) + + if nrows == 0: + logger.info('[OK] Checking for missing or extraneous entries for ' + catname) else: - GV.checkStatus = False - GV.missingEntryStatus = False - logger_with_level = logger.error - log_level = logging.ERROR - - logger.info(('[%s] Checking for missing or extraneous entries for ' + catname) % - ('WARNING' if log_level == logging.WARNING else 'FAIL')) - logger_with_level(' %s has %d issue(s)' % (catname, nrows)) - gplog.log_literal(logger, log_level, " " + " | ".join(fields)) + if catname in ['pg_constraint']: + logger_with_level = logger.warning + log_level = logging.WARNING + else: + GV.checkStatus = False + GV.missingEntryStatus = False + logger_with_level = logger.error + log_level = logging.ERROR + + logger.info(('[%s] Checking for missing or extraneous entries for ' + catname) % + ('WARNING' if log_level == logging.WARNING else 'FAIL')) + logger_with_level(' %s has %d issue(s)' % (catname, nrows)) + gplog.log_literal(logger, log_level, " " + " | ".join(fields)) for row in results: gplog.log_literal(logger, log_level, " " + " | ".join(map(str, row))) processMissingDuplicateEntryResult(catname, fields, results, "missing") if catname == 'pg_type': generateVerifyFile(catname, fields, results, 'missing_extraneous') - return results - except Exception as e: setError(ERROR_NOREPAIR) GV.missingEntryStatus = False @@ -1420,8 +1417,8 @@ def checkTableMissingEntry(cat): class checkAOSegVpinfoThread(execThread): - def __init__(self, cfg, db): - execThread.__init__(self, cfg, db, None) + def __init__(self, cfg, conn): + execThread.__init__(self, cfg, conn, None) def run(self): aoseg_query = """ @@ -1432,15 +1429,17 @@ class checkAOSegVpinfoThread(execThread): try: # Read the list of aoseg tables from the database - curs = self.db.query(aoseg_query) + curs = self.conn.cursor() + curs.execute(aoseg_query) - for relname, relid, segrelid, segrelname in curs.getresult(): + for relname, relid, segrelid, segrelname in curs.fetchall(): qry = "SELECT count(*) FROM pg_attribute WHERE attrelid=%d AND attnum > 0;" % (relid) - attr_count = self.db.query(qry).getresult()[0][0] + curs.execute(qry) + attr_count = curs.fetchone()[0] qry = "SELECT distinct(length(vpinfo)) FROM pg_aoseg.%s where state = 1;" % (segrelname) - vpinfo_curs = self.db.query(qry) - nrows = vpinfo_curs.ntuples() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: continue elif nrows > 1: @@ -1455,7 +1454,7 @@ class checkAOSegVpinfoThread(execThread): logger.error(qry) continue - vpinfo_length = vpinfo_curs.getresult()[0][0] + vpinfo_length = curs.fetchone()[0] # vpinfo is bytea type, the length of the first 3 fields is 12 bytes, and the size of AOCSVPInfoEntry is 16 # typedef struct AOCSVPInfo @@ -1488,8 +1487,8 @@ def checkAOSegVpinfo(): # parallelise check for dbid in GV.cfg: cfg = GV.cfg[dbid] - db_connection = connect2(cfg) - thread = checkAOSegVpinfoThread(cfg, db_connection) + conn = connect2(cfg) + thread = checkAOSegVpinfoThread(cfg, conn) thread.start() logger.debug('launching check thread %s for dbid %i' % (thread.name, dbid)) @@ -1617,9 +1616,10 @@ def checkTableInconsistentEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: logger.info('[OK] Checking for inconsistent entries for ' + catname) @@ -1630,16 +1630,14 @@ def checkTableInconsistentEntry(cat): logger.info('[FAIL] Checking for inconsistent entries for ' + catname) logger.error(' %s has %d issue(s)' % (catname, nrows)) - fields = curs.listfields() + fields = [desc[0] for desc in curs.description] gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): + results = curs.fetchall() + for row in results: gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) - results = curs.getresult() processInconsistentEntryResult(catname, pkey, fields, results) if catname == 'pg_type': generateVerifyFile(catname, fields, results, 'duplicate') - - except Exception as e: setError(ERROR_NOREPAIR) GV.inconsistentEntryStatus = False @@ -1753,9 +1751,10 @@ def checkTableDuplicateEntry(cat): # Execute the query try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - nrows = curs.ntuples() + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + nrows = curs.rowcount if nrows == 0: logger.info('[OK] Checking for duplicate entries for ' + catname) @@ -1767,7 +1766,7 @@ def checkTableDuplicateEntry(cat): fields = curs.listfields() gplog.log_literal(logger, logging.ERROR, " " + " | ".join(fields)) - results = curs.getresult() + results = curs.fetchall() for row in results: gplog.log_literal(logger, logging.ERROR, " " + " | ".join(map(str, row))) processMissingDuplicateEntryResult(catname, fields, results, "duplicate") @@ -1816,9 +1815,9 @@ def duplicateEntryQuery(catname, pkey): def checkUniqueIndexViolation(): logger.info('-----------------------------------') logger.info('Performing check: checking for violated unique indexes') - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - violations = UniqueIndexViolationCheck().runCheck(db_connection) + violations = UniqueIndexViolationCheck().runCheck(conn) checkname = 'unique index violation(s)' if violations: @@ -1855,9 +1854,9 @@ def checkOrphanedToastTables(): logger.info('-----------------------------------') logger.info('Performing check: checking for orphaned TOAST tables') - db_connection = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) checker = OrphanedToastTablesCheck() - check_passed = checker.runCheck(db_connection) + check_passed = checker.runCheck(conn) checkname = 'orphaned toast table(s)' if check_passed: @@ -2376,12 +2375,14 @@ def getOidFromPK(catname, pkeys): pkeystr=pkeystr) try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - if (len(curs.dictresult()) == 0): + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + curs.execute(qry) + results = curs.fetchall() + if (len(results) == 0): raise QueryException("No such entry '%s' in %s" % (pkeystr, catname)) - return curs.dictresult().pop()['oid'] + return results.pop()['oid'] except Exception as e: setError(ERROR_NOREPAIR) @@ -2393,10 +2394,11 @@ def getOidFromPK(catname, pkeys): def getClassOidForRelfilenode(relfilenode): qry = "SELECT oid FROM pg_class WHERE relfilenode = %d;" % (relfilenode) try: - dburl = dbconn.DbURL(hostname=GV.opt['-h'], port=GV.opt['-p'], dbname=GV.dbname) - conn = dbconn.connect(dburl) - oid = dbconn.queryRow(conn, qry)[0] - return oid + with closing(connect()) as conn: + with conn.cursor() as curs: + curs.execute(qry) + oid = curs.fetchone()[0] + return oid except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) @@ -2416,10 +2418,12 @@ def getResourceTypeOid(oid): """ % (oid, oid) try: - db = connect() - curs = db.query(qry) - if len(curs.dictresult()) == 0: return 0 - return curs.dictresult().pop()['oid'] + with closing(connect()) as conn: + with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as curs: + curs.execute(qry) + results = curs.fetchall() + if len(results) == 0: return 0 + return results.pop()['oid'] except Exception as e: setError(ERROR_NOREPAIR) myprint(' Execution error: ' + str(e)) @@ -3014,7 +3018,7 @@ class GPObject: # Collect all tables with missing issues for later reporting if len(self.missingIssues): - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) oid_query = "select (select nspname from pg_namespace where oid=relnamespace) || '.' || relname from pg_class where oid=%d" type_query = "select (select nspname from pg_namespace where oid=relnamespace) || '.' || relname from pg_class where reltype=%d" for issues in self.missingIssues.values() : @@ -3022,19 +3026,23 @@ class GPObject: # Get schemaname.tablename corresponding to oid for key in issue.pkeys: if 'relid' in key or key in ['ev_class', 'reloid']: - table_list = db.query(oid_query % issue.pkeys[key]).getresult() + curs = conn.cursor() + curs.execute(oid_query % issue.pkeys[key]) + table_list = curs.fetchone() if table_list: if issue.type == 'missing': - GV.missing_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.missing_attr_tables.append( (table_list[0], issue.segids) ) else: - GV.extra_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.extra_attr_tables.append( (table_list[0], issue.segids) ) elif key == 'oid': - table_list = db.query(type_query % issue.pkeys[key]).getresult() + curs = conn.cursor() + curs.execute(type_query % issue.pkeys[key]) + table_list = curs.fetchone() if table_list: if issue.type == 'missing': - GV.missing_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.missing_attr_tables.append( (table_list[0], issue.segids) ) else: - GV.extra_attr_tables.append( (table_list[0][0], issue.segids) ) + GV.extra_attr_tables.append( (table_list[0], issue.segids) ) def __cmp__(self, other): @@ -3183,9 +3191,11 @@ def getRelInfo(objects): """.format(oids=','.join(map(str, oids))) try: - db = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) - curs = db.query(qry) - for row in curs.getresult(): + conn = connect2(GV.cfg[GV.coordinator_dbid], utilityMode=False) + curs = conn.cursor() + curs.execute(qry) + results = curs.fetchall() + for row in results: (oid, relname, nspname, relkind, paroid) = row objects[oid, 'pg_class'].setRelInfo(relname, nspname, relkind, paroid) diff --git a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py index b584b9f7557c..eee4b33af2b1 100644 --- a/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/foreign_key_check.py @@ -2,6 +2,7 @@ from gppylib.gplog import * from gppylib.gpcatalog import * +from contextlib import closing import re class ForeignKeyCheck: @@ -116,25 +117,25 @@ def checkTableForeignKey(self, cat): def _validate_relation(self, catname, fkeystr, pkcatname, pkeystr, qry): issue_list = [] try: - curs = self.db_connection.query(qry) - nrows = curs.ntuples() - - if nrows == 0: - self.logger.info('[OK] Foreign key check for %s(%s) referencing %s(%s)' % - (catname, fkeystr, pkcatname, pkeystr)) - else: - self.logger.info('[FAIL] Foreign key check for %s(%s) referencing %s(%s)' % - (catname, fkeystr, pkcatname, pkeystr)) - self.logger.error(' %s has %d issue(s): entry has NULL reference of %s(%s)' % - (catname, nrows, pkcatname, pkeystr)) - - fields = curs.listfields() - log_literal(self.logger, logging.ERROR, " " + " | ".join(fields)) - for row in curs.getresult(): - log_literal(self.logger, logging.ERROR, " " + " | ".join(map(str, row))) - results = curs.getresult() - issue_list.append((pkcatname, fields, results)) - + with closing(self.db_connection.cursor()) as curs: + curs.execute(qry) + nrows = curs.rowcount + + if nrows == 0: + self.logger.info('[OK] Foreign key check for %s(%s) referencing %s(%s)' % + (catname, fkeystr, pkcatname, pkeystr)) + else: + self.logger.info('[FAIL] Foreign key check for %s(%s) referencing %s(%s)' % + (catname, fkeystr, pkcatname, pkeystr)) + self.logger.error(' %s has %d issue(s): entry has NULL reference of %s(%s)' % + (catname, nrows, pkcatname, pkeystr)) + + fields = [desc[0] for desc in curs.description] + log_literal(self.logger, logging.ERROR, " " + " | ".join(fields)) + results = curs.fetchall() + for row in results: + log_literal(self.logger, logging.ERROR, " " + " | ".join(map(str, row))) + issue_list.append((pkcatname, fields, results)) except Exception as e: err_msg = '[ERROR] executing: Foreign key check for catalog table {0}. Query : \n {1}\n'.format(catname, qry) err_msg += str(e) diff --git a/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py b/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py index 87e55a5cf7b7..dc7cfacb32f2 100644 --- a/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py +++ b/gpMgmt/bin/gpcheckcat_modules/leaked_schema_dropper.py @@ -35,16 +35,19 @@ class LeakedSchemaDropper: """ def __get_leaked_schemas(self, db_connection): - leaked_schemas = db_connection.query(self.leaked_schema_query) + with db_connection.cursor() as curs: + curs.execute(self.leaked_schema_query) + leaked_schemas = curs.fetchall() - if not leaked_schemas: - return [] + if not leaked_schemas: + return [] - return [row[0] for row in leaked_schemas.getresult() if row[0]] + return [row[0] for row in leaked_schemas if row[0]] def drop_leaked_schemas(self, db_connection): leaked_schemas = self.__get_leaked_schemas(db_connection) for leaked_schema in leaked_schemas: escaped_schema_name = escapeDoubleQuoteInSQLString(leaked_schema) - db_connection.query('DROP SCHEMA IF EXISTS %s CASCADE;' % (escaped_schema_name)) + with db_connection.cursor() as curs: + curs.execute('DROP SCHEMA IF EXISTS %s CASCADE;' % (escaped_schema_name)) return leaked_schemas diff --git a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py index 21ec8d18047e..a76ef5608672 100644 --- a/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/orphaned_toast_tables_check.py @@ -4,6 +4,8 @@ from collections import namedtuple from gpcheckcat_modules.orphan_toast_table_issues import OrphanToastTableIssue, DoubleOrphanToastTableIssue, ReferenceOrphanToastTableIssue, DependencyOrphanToastTableIssue, MismatchOrphanToastTableIssue +import psycopg2 +from psycopg2 import extras OrphanedTable = namedtuple('OrphanedTable', 'oid catname') @@ -117,8 +119,10 @@ def __init__(self): """ def runCheck(self, db_connection): - orphaned_toast_tables = db_connection.query(self.orphaned_toast_tables_query).dictresult() - if len(orphaned_toast_tables) == 0: + curs = db_connection.cursor(cursor_factory=psycopg2.extras.DictCursor) + curs.execute(self.orphaned_toast_tables_query) + orphaned_toast_tables = curs.fetchall() + if curs.rowcount == 0: return True for row in orphaned_toast_tables: diff --git a/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py b/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py index 47999f5c59c3..6778401f31ac 100644 --- a/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py +++ b/gpMgmt/bin/gpcheckcat_modules/unique_index_violation_check.py @@ -34,22 +34,24 @@ def __init__(self): ) as violations """ - def runCheck(self, db_connection): - unique_indexes = db_connection.query(self.unique_indexes_query).getresult() - violations = [] + def runCheck(self, conn): + with conn.cursor() as cur: + cur.execute(self.unique_indexes_query) + unique_indexes = cur.fetchall() + violations = [] - for (table_oid, index_name, table_name, column_names) in unique_indexes: - column_names = ",".join(column_names) - sql = self.get_violated_segments_query(table_name, column_names) - violated_segments = db_connection.query(sql).getresult() - if violated_segments: - violations.append(dict(table_oid=table_oid, - table_name=table_name, - index_name=index_name, - column_names=column_names, - violated_segments=[row[0] for row in violated_segments])) - - return violations + for (table_oid, index_name, table_name, column_names) in unique_indexes: + column_names = ",".join(column_names) + sql = self.get_violated_segments_query(table_name, column_names) + cur.execute(sql) + violated_segments = cur.fetchall() + if violated_segments: + violations.append(dict(table_oid=table_oid, + table_name=table_name, + index_name=index_name, + column_names=column_names, + violated_segments=[row[0] for row in violated_segments])) + return violations def get_violated_segments_query(self, table_name, column_names): return self.violated_segments_query % ( diff --git a/gpMgmt/bin/gpconfig b/gpMgmt/bin/gpconfig index d6e8defbaf3e..44fdfc7ffc2b 100755 --- a/gpMgmt/bin/gpconfig +++ b/gpMgmt/bin/gpconfig @@ -15,7 +15,7 @@ import os import sys import re - +from psycopg2 import DatabaseError try: from gppylib.gpparseopts import OptParser, OptChecker from gppylib.gparray import GpArray @@ -25,7 +25,6 @@ try: from gppylib.commands.gp import * from gppylib.db import dbconn from gppylib.userinput import * - from pg import DatabaseError from gpconfig_modules.segment_guc import SegmentGuc from gpconfig_modules.database_segment_guc import DatabaseSegmentGuc from gpconfig_modules.file_segment_guc import FileSegmentGuc diff --git a/gpMgmt/bin/gpexpand b/gpMgmt/bin/gpexpand index 89c167be16b1..8ad37dad9bce 100755 --- a/gpMgmt/bin/gpexpand +++ b/gpMgmt/bin/gpexpand @@ -17,10 +17,10 @@ import signal import traceback from collections import defaultdict from time import strftime, sleep - +import psycopg2 +from psycopg2 import DatabaseError, OperationalError +from psycopg2 import extras try: - import pg, pgdb - from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.gparray import GpArray, MODE_NOT_SYNC, STATUS_DOWN @@ -32,7 +32,6 @@ try: from gppylib.operations.startSegments import MIRROR_MODE_MIRRORLESS from gppylib.system import configurationInterface, configurationImplGpdb from gppylib.system.environment import GpCoordinatorEnvironment - from pgdb import DatabaseError from gppylib.gpcatalog import COORDINATOR_ONLY_TABLES from gppylib.operations.package import SyncPackages from gppylib.programs.clsRecoverSegment_triples import get_segments_with_running_basebackup @@ -1880,7 +1879,7 @@ class gpexpand: expansionStopped) dbconn.execSQL(self.conn, sql) self.conn.close() - except pgdb.OperationalError: + except OperationalError: pass except Exception: # schema doesn't exist. Cancel or error during setup @@ -1932,7 +1931,7 @@ class gpexpand: def connect_database(self, dbname): test_url = copy.deepcopy(self.dburl) test_url.pgdb = dbname - c = dbconn.connect(test_url, encoding='UTF8', allowSystemTableMods=True) + c = dbconn.connect(test_url, encoding='UTF8', allowSystemTableMods=True, cursorFactory=psycopg2.extras.NamedTupleCursor) return c def sync_packages(self): @@ -2173,7 +2172,7 @@ class ExpandCommand(SQLCommand): try: status_conn = dbconn.connect(self.status_url, encoding='UTF8') - table_conn = dbconn.connect(self.table_url, encoding='UTF8') + table_conn = dbconn.connect(self.table_url, encoding='UTF8', cursorFactory=psycopg2.extras.NamedTupleCursor) except DatabaseError as ex: if self.options.verbose: logger.exception(ex) diff --git a/gpMgmt/bin/gpload.py b/gpMgmt/bin/gpload.py index 3437dc5ff247..55fe1f78909d 100755 --- a/gpMgmt/bin/gpload.py +++ b/gpMgmt/bin/gpload.py @@ -35,22 +35,8 @@ sys.exit(2) import platform - -try: - import pg -except ImportError: - try: - from pygresql import pg - except Exception as e: - pass -except Exception as e: - print(repr(e)) - errorMsg = "gpload was unable to import The PyGreSQL Python module (pg.py) - %s\n" % str(e) - sys.stderr.write(str(errorMsg)) - errorMsg = "Please check if you have the correct Visual Studio redistributable package installed.\n" - sys.stderr.write(str(errorMsg)) - sys.exit(2) - +import psycopg2 +from psycopg2 import extras import hashlib import datetime,getpass,os,signal,socket,threading,time,traceback,re import subprocess @@ -562,6 +548,8 @@ def is_keyword(tab): else: return False +def escape_string(string): + return psycopg2.extensions.QuotedString(string).getquoted()[1:-1].decode() def caseInsensitiveDictLookup(key, dictionary): """ diff --git a/gpMgmt/bin/gpload_test/Makefile b/gpMgmt/bin/gpload_test/Makefile index 9a6bcec1dbc9..ed8c77c49f7f 100644 --- a/gpMgmt/bin/gpload_test/Makefile +++ b/gpMgmt/bin/gpload_test/Makefile @@ -24,7 +24,7 @@ else installcheck: gpdiff.pl gpstringsubs.pl @echo "doing test in OS: "; echo $(TEST_OS) @cd gpload && ./TEST.py - @cd gpload2 && PYTHONIOENCODING=utf-8 pytest TEST_local_* + @cd gpload2 && export PGSSLMODE=disable && PYTHONIOENCODING=utf-8 pytest TEST_local_* endif clean distclean: diff --git a/gpMgmt/bin/gpload_test/gpload2/data/transform/input_transform.py b/gpMgmt/bin/gpload_test/gpload2/data/transform/input_transform.py index 542fe4bf8809..4936617f44f1 100755 --- a/gpMgmt/bin/gpload_test/gpload2/data/transform/input_transform.py +++ b/gpMgmt/bin/gpload_test/gpload2/data/transform/input_transform.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys import lxml.etree as ET diff --git a/gpMgmt/bin/gpmovemirrors b/gpMgmt/bin/gpmovemirrors index 44b7632be215..ad28a18862be 100755 --- a/gpMgmt/bin/gpmovemirrors +++ b/gpMgmt/bin/gpmovemirrors @@ -10,10 +10,8 @@ import os import sys import signal import itertools - +from psycopg2 import DatabaseError try: - import pg - from gppylib.commands.unix import * from gppylib.commands.gp import * from gppylib.commands.pg import PgControlData @@ -23,7 +21,6 @@ try: from gppylib.db import dbconn from gppylib.userinput import * from gppylib.operations.startSegments import * - from pgdb import DatabaseError from gppylib import gparray, gplog, pgconf, userinput, utils from gppylib.parseutils import line_reader, check_values, canonicalize_address from gppylib.operations.segment_tablespace_locations import get_tablespace_locations diff --git a/gpMgmt/bin/gppylib/commands/base.py b/gpMgmt/bin/gppylib/commands/base.py index c3da5a2aa93a..eba9eb8f4382 100755 --- a/gpMgmt/bin/gppylib/commands/base.py +++ b/gpMgmt/bin/gppylib/commands/base.py @@ -28,7 +28,6 @@ from gppylib import gplog from gppylib import gpsubprocess -from pg import DB logger = gplog.get_default_logger() @@ -628,8 +627,7 @@ def cancel(self): # if self.conn is not set we cannot cancel. if self.cancel_conn: - DB(self.cancel_conn).cancel() - + self.cancel_conn.cancel() def run_remote_commands(name, commands): """ diff --git a/gpMgmt/bin/gppylib/commands/gp.py b/gpMgmt/bin/gppylib/commands/gp.py index b749eb797245..c29d3cee3d67 100644 --- a/gpMgmt/bin/gppylib/commands/gp.py +++ b/gpMgmt/bin/gppylib/commands/gp.py @@ -1009,11 +1009,11 @@ def __init__(self,name,gphome,ctxt=LOCAL,remoteHost=None): self.gphome=gphome #self.cmdStr="%s/bin/postgres --gp-version" % gphome - self.cmdStr="$GPHOME/bin/postgres --gp-version" + self.cmdStr="echo 'START_CMD_OUTPUT';$GPHOME/bin/postgres --gp-version" Command.__init__(self,name,self.cmdStr,ctxt,remoteHost) def get_version(self): - return self.results.stdout.strip() + return self.results.stdout.strip().split('START_CMD_OUTPUT\n')[1] @staticmethod def local(name,gphome): diff --git a/gpMgmt/bin/gppylib/commands/pg.py b/gpMgmt/bin/gppylib/commands/pg.py index 69b629806449..7f2ab9b4e439 100644 --- a/gpMgmt/bin/gppylib/commands/pg.py +++ b/gpMgmt/bin/gppylib/commands/pg.py @@ -14,7 +14,7 @@ from .unix import * from gppylib.commands.base import * from gppylib.commands.gp import RECOVERY_REWIND_APPNAME -from pgdb import DatabaseError +from psycopg2 import DatabaseError logger = get_default_logger() diff --git a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py index 65b49aa424fc..5281e0f8e1a4 100644 --- a/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py +++ b/gpMgmt/bin/gppylib/commands/test/unit/test_unit_pg_base_backup.py @@ -7,7 +7,10 @@ from mock import call, Mock, patch from gppylib.commands import pg from test.unit.gp_unittest import GpTestCase, run_tests -from pgdb import DatabaseError +from psycopg2 import DatabaseError + +from gppylib.test.unit.gp_unittest import GpTestCase +from gppylib.commands.base import CommandResult class TestUnitPgReplicationSlot(GpTestCase): def setUp(self): diff --git a/gpMgmt/bin/gppylib/db/catalog.py b/gpMgmt/bin/gppylib/db/catalog.py index 6214b805c293..80ca66289eb7 100644 --- a/gpMgmt/bin/gppylib/db/catalog.py +++ b/gpMgmt/bin/gppylib/db/catalog.py @@ -6,8 +6,8 @@ """ import copy - -import pg +import os +from contextlib import closing from gppylib import gplog from gppylib.db import dbconn diff --git a/gpMgmt/bin/gppylib/db/dbconn.py b/gpMgmt/bin/gppylib/db/dbconn.py index 1ca0ecf56cb7..24c668d7402b 100644 --- a/gpMgmt/bin/gppylib/db/dbconn.py +++ b/gpMgmt/bin/gppylib/db/dbconn.py @@ -9,9 +9,8 @@ import sys import os import stat - +import psycopg2 try: - import pgdb from gppylib.commands.unix import UserId except ImportError as e: @@ -159,67 +158,44 @@ def canonicalize(s): # 1. pg notice is accessible to a user of connection returned by dbconn.connect(), # lifted from the underlying _pg connection # 2. multiple calls to dbconn.close() should not return an error -class Connection(pgdb.Connection): +class Connection: def __init__(self, connection): - self._notices = collections.deque(maxlen=100) - # we must do an attribute by attribute copy of the notices here - # due to limitations in pg implementation. Wrap with with a - # namedtuple for ease of use. - def handle_notice(notice): - received = {} - for attr in dir(notice): - if attr.startswith('__'): - continue - value = getattr(notice, attr) - received[attr] = value - Notice = collections.namedtuple('Notice', sorted(received)) - self._notices.append(Notice(**received)) - - - self._impl = connection - self._impl._cnx.set_notice_receiver(handle_notice) + self._conn = connection + self._conn.notices = collections.deque(maxlen=100) def __enter__(self): - return self._impl.__enter__() + return self._conn.__enter__() # __exit__() does not close the connection. This is in line with the # python DB API v2 specification (pep-0249), where close() is done on # __del__(), not __exit__(). def __exit__(self, *args): - return self._impl.__exit__(*args) + return self._conn.__exit__(*args) def __getattr__(self, name): - return getattr(self._impl, name) + return getattr(self._conn, name) def notices(self): - notice_list = list(self._notices) - self._notices.clear() + notice_list = list(self._conn.notices) + self._conn.notices.clear() return notice_list # don't return operational error if connection is already closed def close(self): - if not self._impl.closed: - self._impl.close() + if not self._conn.closed: + self._conn.close() def connect(dburl, utility=False, verbose=False, - encoding=None, allowSystemTableMods=False, logConn=True, unsetSearchPath=True): + encoding=None, allowSystemTableMods=False, logConn=True, unsetSearchPath=True, cursorFactory=None): conninfo = { 'user': dburl.pguser, 'password': dburl.pgpass, 'host': dburl.pghost, 'port': dburl.pgport, - # dbname is very subtle, Package pgdb contains a bug it will only escape the string when - # 1. a space in the dbname, and - # 2. there are other keyword arguments of pgdb.connect method - # See issue https://github.com/PyGreSQL/PyGreSQL/issues/77 for details - # The code here is test if there is space, if so, we know pgdb will escape, let's not do here - # if not, let's do escape here since pgdb forget to do. - # - # NB: we always provide port keyword argument to connect method of pgdb, thus - # we will always enter the code path of pgdb.connect of the above escape logic. - 'database': dburl.pgdb if ' ' in dburl.pgdb else dburl.pgdb.replace('\\', '\\\\').replace("'", "\\'"), + 'database': dburl.pgdb, + 'cursor_factory': cursorFactory } # building options @@ -257,22 +233,23 @@ def connect(dburl, utility=False, verbose=False, logFunc = logger.info if dburl.timeout is not None else logger.debug logFunc("Connecting to db {} on host {}".format(dburl.pgdb, dburl.pghost)) - connection = None + conn = None for i in range(retries): try: - connection = pgdb.connect(**conninfo) + conn = psycopg2.connect(**conninfo) + conn.set_session(autocommit=True) break - except pgdb.OperationalError as e: + except psycopg2.OperationalError as e: if 'timeout expired' in str(e): logger.warning('Timeout expired connecting to %s, attempt %d/%d' % (dburl.pgdb, i+1, retries)) continue raise - if connection is None: + if conn is None: raise ConnectionError('Failed to connect to %s' % dburl.pgdb) - return Connection(connection) + return Connection(conn) def execSQL(conn, sql, autocommit=True): """ @@ -286,7 +263,6 @@ def execSQL(conn, sql, autocommit=True): Using `with dbconn.connect() as conn` syntax will override autocommit and complete queries in a transaction followed by a commit on context close """ - conn.autocommit = autocommit with conn.cursor() as cursor: cursor.execute(sql) diff --git a/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py b/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py index 086ccce92981..a03eb4db42b6 100644 --- a/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py +++ b/gpMgmt/bin/gppylib/db/test/unit/test_cluster_dbconn.py @@ -46,7 +46,7 @@ def test_verbose_mode_allows_warnings_to_be_sent_to_the_client(self): for notice in notices: - if warning in notice.message: + if warning in notice: return # found it! self.fail("Didn't find expected notice '{}' in {!r}".format( diff --git a/gpMgmt/bin/gppylib/gpcatalog.py b/gpMgmt/bin/gppylib/gpcatalog.py index 81b6e5f02e9d..fc2916ac75e4 100644 --- a/gpMgmt/bin/gppylib/gpcatalog.py +++ b/gpMgmt/bin/gppylib/gpcatalog.py @@ -143,7 +143,7 @@ def __init__(self, dbConnection): curs = self._query(version_query) except Exception as e: raise GPCatalogException("Error reading database version: " + str(e)) - self._version = GpVersion(curs.getresult()[0][0]) + self._version = GpVersion(curs.fetchone()[0]) # Read the list of catalog tables from the database try: @@ -153,7 +153,7 @@ def __init__(self, dbConnection): # Construct our internal representation of the catalog - for [oid, relname, relisshared] in curs.getresult(): + for [oid, relname, relisshared] in curs.fetchall(): self._tables[relname] = GPCatalogTable(self, relname) # Note: stupid API returns t/f for boolean value self._tables[relname]._setShared(relisshared == 't') @@ -192,7 +192,9 @@ def _query(self, qry): """ Simple wrapper around querying the database connection """ - return self._dbConnection.query(qry) + cur = self._dbConnection.cursor() + cur.execute(qry) + return cur def _markCoordinatorOnlyTables(self): """ @@ -482,10 +484,10 @@ def __init__(self, parent, name, pkey=None): # exist. raise GPCatalogException("Catalog table %s does not exist" % name) - if cur.ntuples() == 0: + if cur.rowcount == 0: raise GPCatalogException("Catalog table %s does not exist" % name) - for row in cur.getresult(): + for row in cur.fetchall(): (attname, atttype, typname) = row # Mark if the catalog has an oid column @@ -521,7 +523,7 @@ def __init__(self, parent, name, pkey=None): WHERE attrelid = 'pg_catalog.{catname}'::regclass """.format(catname=name) cur = parent._query(qry) - self._pkey = [row[0] for row in cur.getresult()] + self._pkey = [row[0] for row in cur.fetchall()] # Primary key must be in the column list for k in self._pkey: diff --git a/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py b/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py index 49dd622bb628..fac01fd3f513 100644 --- a/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py +++ b/gpMgmt/bin/gppylib/operations/segment_reconfigurer.py @@ -1,9 +1,7 @@ import time - from gppylib.commands import base from gppylib.db import dbconn -import pg - +from contextlib import closing FTS_PROBE_QUERY = 'SELECT pg_catalog.gp_request_fts_probe_scan()' @@ -14,15 +12,19 @@ def __init__(self, logger, worker_pool, timeout): self.timeout = timeout def _trigger_fts_probe(self, dburl): - conn = pg.connect(dbname=dburl.pgdb, - host=dburl.pghost, - port=dburl.pgport, - opt=None, - user=dburl.pguser, - passwd=dburl.pgpass, - ) - conn.query(FTS_PROBE_QUERY) - conn.close() + start_time = time.time() + while True: + try: + with closing(dbconn.connect(dburl)) as conn: + with conn.cursor() as cur: + cur.execute(FTS_PROBE_QUERY) + break + except Exception as e: + now = time.time() + if now < start_time + self.timeout: + continue + else: + raise RuntimeError("FTS probing did not complete in {} seconds.".format(self.timeout)) def reconfigure(self): # issue a distributed query to make sure we pick up the fault @@ -36,15 +38,15 @@ def reconfigure(self): # Empty block of 'BEGIN' and 'END' won't start a distributed transaction, # execute a DDL query to start a distributed transaction. # so the primaries'd better be up - conn = dbconn.connect(dburl) - conn.cursor().execute('CREATE TEMP TABLE temp_test(a int)') - conn.cursor().execute('COMMIT') + with closing(dbconn.connect(dburl)) as conn: + with conn.cursor() as cur: + cur.execute('BEGIN') + cur.execute('CREATE TEMP TABLE temp_test(a int)') + cur.execute('COMMIT') + break except Exception as e: now = time.time() if now < start_time + self.timeout: continue else: raise RuntimeError("Mirror promotion did not complete in {0} seconds.".format(self.timeout)) - else: - conn.close() - break diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py index 9d86071bafd7..d4c2a0c26e72 100644 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_segment_reconfigurer.py @@ -4,8 +4,7 @@ from gppylib.operations.segment_reconfigurer import SegmentReconfigurer, FTS_PROBE_QUERY from gppylib.test.unit.gp_unittest import GpTestCase -import pg -import pgdb +import psycopg2 import mock from mock import Mock, patch, call, MagicMock import contextlib @@ -38,22 +37,22 @@ def setUp(self): self.apply_patches([ patch('gppylib.db.dbconn.connect', new=self.connect), patch('gppylib.db.dbconn.DbURL', return_value=self.db_url), - patch('pg.connect'), + patch('psycopg2.connect'), ]) def test_it_triggers_fts_probe(self): reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=self.worker_pool, timeout=self.timeout) reconfigurer.reconfigure() - pg.connect.assert_has_calls([ - call(dbname=self.db, host=self.host, port=self.port, opt=None, user=self.user, passwd=self.passwd), + psycopg2.connect.assert_has_calls([ + call(dbname=self.db, host=self.host, port=self.port, options=None, user=self.user, password=self.passwd), call().query(FTS_PROBE_QUERY), call().close(), ] ) def test_it_retries_the_connection(self): - self.connect.configure_mock(side_effect=[pgdb.DatabaseError, pgdb.DatabaseError, self.conn]) + self.connect.configure_mock(side_effect=[psycopg2.DatabaseError, psycopg2.DatabaseError, self.conn]) reconfigurer = SegmentReconfigurer(logger=self.logger, worker_pool=self.worker_pool, timeout=self.timeout) @@ -74,7 +73,7 @@ def fail_for_five_minutes(): # leap forward 300 seconds new_time += self.timeout / 2 now_mock.configure_mock(return_value=new_time) - yield pgdb.DatabaseError + yield psycopg2.DatabaseError self.connect.configure_mock(side_effect=fail_for_five_minutes()) @@ -87,3 +86,27 @@ def fail_for_five_minutes(): self.connect.assert_has_calls([call(self.db_url), call(self.db_url), ]) self.conn.close.assert_has_calls([]) + + @patch('time.time') + def test_it_gives_up_after_600_seconds_2(self, now_mock): + start_datetime = datetime.datetime(2023, 7, 27, 16, 0, 0) + start_time = time.mktime(start_datetime.timetuple()) + now_mock.configure_mock(return_value=start_time) + + def fail_for_ten_minutes(): + new_time = start_time + # leap forward 600 seconds + new_time += self.timeout + now_mock.configure_mock(return_value=new_time) + yield psycopg2.DatabaseError + + self.connect.configure_mock(side_effect=fail_for_ten_minutes()) + + reconfigurer = SegmentReconfigurer(logger=self.logger, + worker_pool=self.worker_pool, timeout=self.timeout) + with self.assertRaises(RuntimeError) as context: + reconfigurer.reconfigure() + self.assertEqual("FTS probing did not complete in {} seconds.".format(self.timeout), context.exception.message) + + self.connect.assert_has_calls([call(self.db_url)]) + self.conn.close.assert_has_calls([]) diff --git a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py index 38fcd327dee4..e62aa7a40b72 100755 --- a/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py +++ b/gpMgmt/bin/gppylib/operations/test/unit/test_unit_utils.py @@ -11,7 +11,7 @@ from gppylib.operations.test_utils_helper import TestOperation, RaiseOperation, RaiseOperation_Unpicklable, RaiseOperation_Safe, ExceptionWithArgs from operations.unix import ListFiles from test.unit.gp_unittest import GpTestCase, run_tests -from pg import DatabaseError +from psycopg2 import DatabaseError class UtilsTestCase(GpTestCase): """ diff --git a/gpMgmt/bin/gppylib/operations/test_utils_helper.py b/gpMgmt/bin/gppylib/operations/test_utils_helper.py index 109bc83bb878..74cfeda2bc01 100755 --- a/gpMgmt/bin/gppylib/operations/test_utils_helper.py +++ b/gpMgmt/bin/gppylib/operations/test_utils_helper.py @@ -1,4 +1,5 @@ from gppylib.operations import Operation +import psycopg2 """ These objects needed for gppylib.operations.test.test_utils are pulled out of said file for @@ -37,5 +38,4 @@ def __init__(self, x, y): class RaiseOperation_Unpicklable(Operation): def execute(self): - import pg - raise pg.DatabaseError() + raise psycopg2.DatabaseError() diff --git a/gpMgmt/bin/gppylib/programs/clsSystemState.py b/gpMgmt/bin/gppylib/programs/clsSystemState.py index 5c120a592a47..62aba744904f 100644 --- a/gpMgmt/bin/gppylib/programs/clsSystemState.py +++ b/gpMgmt/bin/gppylib/programs/clsSystemState.py @@ -10,7 +10,7 @@ import sys, os import re import collections -import pgdb +import psycopg2 from contextlib import closing from gppylib import gparray, gplog from gppylib.commands import base, gp @@ -1026,7 +1026,7 @@ def _get_unsync_segs_add_wal_remaining_bytes(data, gpArray): wal_sync_bytes_out = 'Unknown' unsync_segs.append(s) data.addValue(VALUE__REPL_SYNC_REMAINING_BYTES, wal_sync_bytes_out) - except pgdb.InternalError: + except (psycopg2.InternalError, psycopg2.OperationalError): logger.warning('could not query segment {} ({}:{})'.format( s.dbid, s.hostname, s.port )) @@ -1098,7 +1098,7 @@ def _add_replication_info(data, primary, mirror): cursor.close() - except pgdb.InternalError: + except (psycopg2.InternalError, psycopg2.OperationalError): logger.warning('could not query segment {} ({}:{})'.format( primary.dbid, primary.hostname, primary.port )) diff --git a/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py b/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py index 48418d8b7bbc..0c62db259369 100644 --- a/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py +++ b/gpMgmt/bin/gppylib/programs/test/unit/test_cluster_clsrecoversegment_triples.py @@ -194,7 +194,7 @@ def test_RecoveryTripletsUserConfigFile_getMirrorTriples_should_fail(self): }, { "name": "failed_and_live_same_dbid", - "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + "gparray": """1|-1|p|p|n|u|cdw|cdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -208,7 +208,7 @@ def test_RecoveryTripletsUserConfigFile_getMirrorTriples_should_fail(self): }, { "name": "failover_unreachable", - "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + "gparray": """1|-1|p|p|n|u|cdw|cdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -307,7 +307,7 @@ def test_RecoveryTripletsInPlaceAndNewHosts_getMirrorTriples_should_pass(self): }, { "name": "failed_unreachable", - "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -414,7 +414,7 @@ def test_RecoveryTripletsInPlaceAndNewHosts_getMirrorTriples_should_fail(self): }, { "name": "failed_and_live_same_dbid", - "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -428,7 +428,7 @@ def test_RecoveryTripletsInPlaceAndNewHosts_getMirrorTriples_should_fail(self): }, { "name": "failover_unreachable", - "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + "gparray": """1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -505,7 +505,7 @@ def __init__(self, arg): super().__init__(arg) # It's possible to have no down segments, as gpmovemirrors also calls gprecoverseg. - self.all_up_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + self.all_up_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|p|p|s|u|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 16|6|m|m|s|u|sdw1|sdw1|21000|/mirror/gpseg6 @@ -524,7 +524,7 @@ def __init__(self, arg): 9|7|p|p|s|u|sdw4|sdw4|20001|/primary/gpseg7''' # We include down segments, so that gprecoverseg can find them automatically - self.three_failedover_segs_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + self.three_failedover_segs_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|m|p|s|d|sdw1|sdw1|20001|/primary/gpseg1 8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -534,7 +534,7 @@ def __init__(self, arg): 4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2 5|3|p|p|s|u|sdw2|sdw2|20001|/primary/gpseg3''' - self.content0_no_peer_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + self.content0_no_peer_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|m|p|s|d|sdw1|sdw1|20001|/primary/gpseg1 8|2|p|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -543,7 +543,7 @@ def __init__(self, arg): 4|2|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg2 5|3|p|p|s|u|sdw2|sdw2|20001|/primary/gpseg3''' - self.content0_mirror_and_its_peer_down_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + self.content0_mirror_and_its_peer_down_gparray_str = '''1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw1|sdw1|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 @@ -553,7 +553,7 @@ def __init__(self, arg): 4|2|p|p|s|u|sdw2|sdw2|20000|/primary/gpseg2 5|3|p|p|s|u|sdw2|sdw2|20001|/primary/gpseg3''' - self.content0_mirror_and_its_peer_down_gparray_str2 = '''1|-1|p|p|n|u|mdw|mdw|5432|/master/gpseg-1 + self.content0_mirror_and_its_peer_down_gparray_str2 = '''1|-1|p|p|n|u|mdw|mdw|5432|/coordinator/gpseg-1 2|0|m|p|s|d|sdw2|sdw2|20000|/primary/gpseg0 3|1|p|p|s|u|sdw1|sdw1|20001|/primary/gpseg1 8|2|m|m|s|u|sdw3|sdw3|21000|/mirror/gpseg2 diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py b/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py index 9888472bb436..b9e691a7e567 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_foreign_key_check.py @@ -11,7 +11,7 @@ class GpCheckCatTestCase(GpTestCase): def setUp(self): self.logger = Mock(spec=['log', 'info', 'debug', 'error']) - self.db_connection = Mock(spec=['close', 'query']) + self.db_connection = Mock(spec=['close', 'cursor']) self.autoCast = {'regproc': '::oid', 'regprocedure': '::oid', 'regoper': '::oid', @@ -25,9 +25,10 @@ def setUp(self): self.full_join_cat_tables = set(['pg_attribute','gp_distribution_policy','pg_appendonly','pg_constraint','pg_index']) self.foreign_key_check= Mock(spec=['runCheck']) self.foreign_key_check.runCheck.return_value = [] - self.db_connection.query.return_value.ntuples.return_value = 2 - self.db_connection.query.return_value.listfields.return_value = ['pkey1', 'pkey2'] - self.db_connection.query.return_value.getresult.return_value = [('r1','r2'), ('r3','r4')] + + self.db_connection.cursor.return_value.rowcount = 2 + self.db_connection.cursor.return_value.description = [('pkey1',), ('pkey2',)] + self.db_connection.cursor.return_value.fetchall.return_value = [('r1','r2'), ('r3','r4')] def test_get_fk_query_left_join_returns_the_correct_query(self): @@ -127,7 +128,7 @@ def test_checkTableForeignKey__returns_correct_join_query(self, log_literal_mock self.assertEqual(len(issue_list) , 2) self.assertEqual(issue_list[0], ('pg_class', ['pkey1', 'pkey2'], [('r1', 'r2'), ('r3', 'r4')])) self.assertEqual(issue_list[1], ('arbitrary_catalog_table', ['pkey1', 'pkey2'], [('r1', 'r2'), ('r3', 'r4')])) - self.assertEqual(self.db_connection.query.call_count, 2) + self.assertEqual(self.db_connection.cursor.call_count, 2) def __generate_pg_class_call(table, primary_key_cat_name, col_type, with_filter=True): if with_filter: @@ -168,7 +169,7 @@ def __generate_pg_class_call(table, primary_key_cat_name, col_type, with_filter= self.assertEqual(fk_query_full_join_mock.call_count, 0) fk_query_left_join_mock.assert_has_calls(foreign_key_mock_calls_left, any_order=False) - self.db_connection.query.call_count = 0 + self.db_connection.cursor.call_count = 0 fk_query_full_join_mock.call_count = 0 fk_query_left_join_mock.call_count = 0 diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py index 63f2d88fe89f..28aa0f1227c5 100755 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpcheckcat.py @@ -18,11 +18,11 @@ def setUp(self): self.subject = imp.load_source('gpcheckcat', gpcheckcat_file) self.subject.check_gpexpand = lambda : (True, "") - self.db_connection = Mock(spec=['close', 'query']) + self.db_connection = Mock(spec=['close', 'cursor', 'set_session']) self.unique_index_violation_check = Mock(spec=['runCheck']) self.foreign_key_check = Mock(spec=['runCheck', 'checkTableForeignKey']) self.apply_patches([ - patch("gpcheckcat.pg.connect", return_value=self.db_connection), + patch("gpcheckcat.connect", return_value=self.db_connection), patch("gpcheckcat.UniqueIndexViolationCheck", return_value=self.unique_index_violation_check), patch("gpcheckcat.ForeignKeyCheck", return_value=self.foreign_key_check), patch('os.environ', new={}), @@ -129,23 +129,26 @@ def test_drop_leaked_schemas__when_leaked_schemas_exist__reports_which_schemas_a self.assertIn(expected_message, log_messages) def test_automatic_thread_count(self): - self.db_connection.query.return_value.getresult.return_value = [[0]] + self.db_connection.cursor.return_value.fetchall.return_value = [[0]] self._run_batch_size_experiment(100) self._run_batch_size_experiment(101) + @patch('gpcheckcat.getversion', return_value='4.3') @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @patch('gppylib.gplog.log_literal') - def test_truncate_batch_size(self, mock_log, mock_gpcheckcat, mock_sys_exit): + def test_truncate_batch_size(self, mock_log, mock_sys_exit, mock_gpcatalog, mock_version): self.subject.GV.opt['-B'] = 300 # override the setting from available memory # setup conditions for 50 primaries and plenty of RAM such that max threads > 50 primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.getresult.return_value = [['4.3']] - self.db_connection.query.return_value.dictresult.return_value = primaries + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries testargs = ['some_string','-port 1', '-R foo'] @@ -221,10 +224,11 @@ def test_checkForeignKey__no_arg(self, process_foreign_key_mock): self.foreign_key_check.runCheck.assert_called_once_with(cat_tables) # Test gpcheckat -C option with checkForeignKey + @patch('gpcheckcat.getversion', return_value='4.3') @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @patch('gpcheckcat.checkTableMissingEntry') - def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3): + def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3, mock4): self.subject.checkForeignKey = Mock() gpcat_class_mock = Mock(spec=['getCatalogTable']) cat_obj_mock = Mock() @@ -234,8 +238,12 @@ def test_runCheckCatname__for_checkForeignKey(self, mock1, mock2, mock3): for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.getresult.return_value = [['4.3']] - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries self.subject.GV.opt['-C'] = 'pg_class' @@ -314,13 +322,19 @@ def test_skip_one_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s test2'] with patch.object(sys, 'argv', testargs): self.subject.main() - mock_run.assert_has_calls(call(['test1', 'test3'])) + mock_run.assert_has_calls([call(['test1', 'test3'])]) @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @@ -330,13 +344,19 @@ def test_skip_multiple_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s', "test1, test2"] with patch.object(sys, 'argv', testargs): self.subject.main() - mock_run.assert_has_calls(call(['test3'])) + mock_run.assert_has_calls([call(['test3'])]) @patch('gpcheckcat.GPCatalog', return_value=Mock()) @patch('sys.exit') @@ -346,13 +366,17 @@ def test_skip_test_warning(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-s', "test_invalid, test2"] with patch.object(sys, 'argv', testargs): self.subject.main() - mock_run.assert_has_calls(call(['test1', 'test3'])) + mock_run.assert_has_calls([call(['test1', 'test3'])]) expected_message = "'test_invalid' is not a valid test" log_messages = [args[0][1] for args in self.subject.logger.log.call_args_list] self.assertIn(expected_message, log_messages) @@ -365,7 +389,13 @@ def test_run_multiple_test(self, mock_ver, mock_run, mock1, mock2): primaries = [dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=-1, dbid=0, isprimary='t')] for i in range(1, 50): primaries.append(dict(hostname='host0', port=123, id=1, address='123', datadir='dir', content=1, dbid=i, isprimary='t')) - self.db_connection.query.return_value.dictresult.return_value = primaries + + # context manager helper functions. + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = primaries + self.subject.all_checks = {'test1': 'a', 'test2': 'b', 'test3': 'c'} testargs = ['gpcheckcat', '-port 1', '-R', "test1, test2"] diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py index 298cf80d61c1..a3e3bd5c2026 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpconfig.py @@ -5,11 +5,10 @@ import shutil import sys import tempfile - +from psycopg2 import DatabaseError from gppylib.gparray import Segment, GpArray, SegmentPair from gpconfig_modules.parse_guc_metadata import ParseGuc import errno -from pg import DatabaseError from .gp_unittest import * from unittest.mock import * diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py b/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py index e9c60760aebb..4858e80e65a4 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_gpstate.py @@ -1,6 +1,6 @@ import unittest import mock -import pgdb +import psycopg2 import tempfile from gppylib import gparray @@ -228,7 +228,7 @@ def test_add_replication_info_adds_unknowns_if_primary_is_down(self): @mock.patch('gppylib.db.dbconn.connect', autospec=True) def test_add_replication_info_adds_unknowns_if_connection_cannot_be_made(self, mock_connect): # Simulate a connection failure in dbconn.connect(). - mock_connect.side_effect = pgdb.InternalError('connection failure forced by unit test') + mock_connect.side_effect = psycopg2.InternalError('connection failure forced by unit test') GpSystemStateProgram._add_replication_info(self.data, self.primary, self.mirror) self.assertEqual('Unknown', self.data.getStrValue(self.mirror, VALUE__REPL_SENT_LSN)) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py b/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py index 1eda2e72a250..1cd81684c235 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_leaked_schema_dropper.py @@ -6,45 +6,57 @@ class LeakedSchemaDropperTestCase(GpTestCase): def setUp(self): - self.db_connection = Mock(spec=['query']) - - two_leaked_schemas = Mock() - two_leaked_schemas.getresult.return_value = [ + self.db_connection = Mock(spec=['cursor']) + self.db_connection.cursor.return_value.fetchall.return_value = [ ('fake_leak_1', 'something_else'), ('some"test"special_#;character--schema', 'something_else') ] - self.db_connection.query.return_value = two_leaked_schemas - self.subject = LeakedSchemaDropper() def test_drop_leaked_schemas__returns_a_list_of_leaked_schemas(self): + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [ + ('fake_leak_1', 'something_else'), + ('some"test"special_#;character--schema', 'something_else') + ] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), ['fake_leak_1', 'some"test"special_#;character--schema']) def test_drop_leaked_schemas__when_there_are_no_leaked_schemas__returns_an_empty_list(self): - no_leaked_schemas = Mock() - no_leaked_schemas.getresult.return_value = [] - self.db_connection.query.return_value = no_leaked_schemas - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__when_query_returns_null_schema__returns_an_empty_list(self): - null_leaked_schema = Mock() - null_leaked_schema.getresult.return_value = [(None, 'something_else')] - self.db_connection.query.return_value = null_leaked_schema - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [(None, 'something_else')] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__when_query_returns_null__returns_an_empty_list(self): - self.db_connection.query.return_value = None - + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [] self.assertEqual(self.subject.drop_leaked_schemas(self.db_connection), []) def test_drop_leaked_schemas__drops_orphaned_and_leaked_schemas(self): + self.db_connection.cursor.return_value = Mock() + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.return_value = [ + ('fake_leak_1', 'something_else'), + ('some"test"special_#;character--schema', 'something_else') + ] self.subject.drop_leaked_schemas(self.db_connection) drop_query_expected_list = [call("DROP SCHEMA IF EXISTS \"fake_leak_1\" CASCADE;"), call("DROP SCHEMA IF EXISTS \"some\"\"test\"\"special_#;character--schema\" CASCADE;")] - self.db_connection.query.assert_has_calls(drop_query_expected_list) + self.db_connection.cursor.return_value.__enter__.return_value.execute.assert_has_calls(drop_query_expected_list) if __name__ == '__main__': diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py b/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py index 8d7f7c7a316c..a2b0773be085 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_recovery_base.py @@ -3,6 +3,7 @@ import io from mock import call, Mock, patch, ANY import sys +import re from .gp_unittest import GpTestCase import gppylib @@ -60,8 +61,8 @@ def _asserts_for_passing_tests(self, stderr_buf, ex, enable_verbose_count=0, war def _asserts_for_failing_tests(self, ex, stderr_buf, expected_message, info_count=1): self.assertEqual(1, ex.exception.code) - self.assertEqual(expected_message, stderr_buf.getvalue().strip()) - self.assertEqual([call(expected_message)], self.mock_logger.error.call_args_list) + self.assertRegexpMatches(expected_message, stderr_buf.getvalue().strip()) + self.assertTrue(any(re.search(expected_message, call_args[0]) for call_args, _ in self.mock_logger.error.call_args_list)) self.assertEqual(info_count, self.mock_logger.info.call_count) def _assert_workerpool_calls(self, mock_workerpool): @@ -211,9 +212,9 @@ def test_invalid_cmd_fails(self): stderr_buf, ex = self.run_recovery_base_get_stderr() self._asserts_for_failing_tests(ex, stderr_buf, - '[{"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + '[{"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null},' - ' {"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + ' {"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null}]') #TODO do we need this test where an invalid command fails but with a wrapper error? @@ -239,9 +240,9 @@ def test_invalid_cmd_verbose_fails(self): stderr_buf, ex = self.run_recovery_base_get_stderr() self._asserts_for_failing_tests(ex, stderr_buf, - '[{"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + '[{"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null},' - ' {"error_type": "default", "error_msg": "/bin/bash: invalid_cmd_str: command not found\\n",' + ' {"error_type": "default", "error_msg": "^/bin/bash.*invalid_cmd_str: command not found$",' ' "dbid": null, "datadir": null, "port": null, "progress_file": null}]') self.assertEqual(1, self.mock_enable_verbose_logging.call_count) diff --git a/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py b/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py index c9306d69b97c..19ddb7732324 100644 --- a/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py +++ b/gpMgmt/bin/gppylib/test/unit/test_unit_unique_index_violation_check.py @@ -9,33 +9,37 @@ def setUp(self): self.subject = UniqueIndexViolationCheck() self.index_query_result = Mock() - self.index_query_result.getresult.return_value = [ + self.index_query_result.fetchall.return_value = [ (9001, 'index1', 'table1', ['index1_column1','index1_column2']), (9001, 'index2', 'table1', ['index2_column1','index2_column2']) ] - self.violated_segments_query_result = Mock() - - self.db_connection = Mock(spec=['query']) - self.db_connection.query.side_effect = self.mock_query_return_value - - def mock_query_return_value(self, query_string): - if query_string == UniqueIndexViolationCheck.unique_indexes_query: - return self.index_query_result - else: - return self.violated_segments_query_result + self.db_connection = Mock(spec=['cursor']) + self.db_connection.cursor.return_value.__enter__ = Mock(return_value=Mock(spec=['fetchall', 'execute'])) + self.db_connection.cursor.return_value.__exit__ = Mock(return_value=False) def test_run_check__when_there_are_no_issues(self): - self.violated_segments_query_result.getresult.return_value = [] + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.side_effect = [ + [ + (9001, 'index1', 'table1', ['index1_column1','index1_column2']), + (9001, 'index2', 'table1', ['index2_column1','index2_column2']) + ], + [], + [], + ] violations = self.subject.runCheck(self.db_connection) self.assertEqual(len(violations), 0) def test_run_check__when_index_is_violated(self): - self.violated_segments_query_result.getresult.side_effect = [ + self.db_connection.cursor.return_value.__enter__.return_value.fetchall.side_effect = [ + [ + (9001, 'index1', 'table1', ['index1_column1','index1_column2']), + (9001, 'index2', 'table1', ['index2_column1','index2_column2']) + ], [(-1,), (0,), (1,)], - [(-1,)] + [(-1,)], ] violations = self.subject.runCheck(self.db_connection) diff --git a/gpMgmt/bin/gppylib/utils.py b/gpMgmt/bin/gppylib/utils.py index fead818d3ac1..42c77e620ab3 100644 --- a/gpMgmt/bin/gppylib/utils.py +++ b/gpMgmt/bin/gppylib/utils.py @@ -5,8 +5,7 @@ from sys import * from xml.dom import minidom from xml.dom import Node - -import pgdb +import psycopg2 from gppylib.gplog import * logger = get_default_logger() @@ -503,14 +502,12 @@ def escapeDoubleQuoteInSQLString(string, forceDoubleQuote=True): string = '"' + string + '"' return string - -def Escape(query_str): - return pgdb.escape_string(query_str) - +def escape_string(string): + return psycopg2.extensions.QuotedString(string).getquoted()[1:-1].decode() def escapeArrayElement(query_str): # also escape backslashes and double quotes, in addition to the doubling of single quotes - return pgdb.escape_string(query_str.encode(errors='backslashreplace')).decode(errors='backslashreplace').replace('\\','\\\\').replace('"','\\"') + return escape_string(query_str.encode(errors='backslashreplace')).encode().decode(errors='backslashreplace').replace('\\','\\\\').replace('"','\\"') # Transform Python list to Postgres array literal (of the form: '{...}') @@ -593,7 +590,7 @@ def formatInsertValuesList(row, starelid, inclHLL): # Format stavalues5 for an hll slot elif i == 30 and hll: if inclHLL: - val = '\'{"%s"}\'' % pgdb.escape_bytea(val[0]) + val = '\'{\\%s}\'' % val[0] rowVals.append('\t{0}::{1}'.format(val, 'bytea[]')) else: rowVals.append('\t{0}'.format('NULL::int4[]')) diff --git a/gpMgmt/bin/gpsd b/gpMgmt/bin/gpsd index daa8146ab913..363ed9d46ed8 100755 --- a/gpMgmt/bin/gpsd +++ b/gpMgmt/bin/gpsd @@ -12,8 +12,8 @@ import re from contextlib import closing from distutils.version import LooseVersion from optparse import OptionParser -import pgdb -from gppylib.utils import formatInsertValuesList, Escape +from gppylib.utils import formatInsertValuesList, escape_string +import psycopg2 gpsd_version = '%prog 1.0' @@ -43,7 +43,7 @@ def get_num_segments(cursor): query = "select count(*) from gp_segment_configuration where role='p' and content >=0;" try: cursor.execute(query) - except pgdb.DatabaseError as e: + except psycopg2.DatabaseError as e: sys.stderr.write('\nError while trying to retrieve number of segments.\n\n' + str(e) + '\n\n') sys.exit(1) vals = cursor.fetchone() @@ -83,7 +83,13 @@ def dumpTupleCount(cur): def dumpStats(cur, inclHLL): - query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, pgs.* ' \ + query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, ' \ + 'pgs.starelid, pgs.staattnum, pgs.stainherit, pgs.stanullfrac, pgs.stawidth, pgs.stadistinct, ' \ + 'pgs.stakind1, pgs.stakind2, pgs.stakind3, pgs.stakind4, pgs.stakind5, ' \ + 'pgs.staop1, pgs.staop2, pgs.staop3, pgs.staop4, pgs.staop5, ' \ + 'pgs.stacoll1, pgs.stacoll2, pgs.stacoll3, pgs.stacoll4, pgs.stacoll5, ' \ + 'pgs.stanumbers1, pgs.stanumbers2, pgs.stanumbers3, pgs.stanumbers4, pgs.stanumbers5, ' \ + 'pgs.stavalues1::text::text[], pgs.stavalues2::text::text[], pgs.stavalues3::text::text[], pgs.stavalues4::text::text[], pgs.stavalues5::text::text[] ' \ 'FROM pg_class pgc, pg_statistic pgs, pg_namespace pgn, pg_attribute pga, pg_type pgt, pg_namespace pgtn ' \ 'WHERE pgc.relnamespace = pgn.oid and pgn.nspname NOT IN ' + \ sysnslist + \ @@ -102,7 +108,7 @@ def dumpStats(cur, inclHLL): cur.execute(query) for vals in ResultIter(cur): - starelid = "'%s.%s'::regclass" % (Escape(vals[1]), Escape(vals[0])) + starelid = "'%s.%s'::regclass" % (escape_string(vals[1]), escape_string(vals[0])) rowVals = formatInsertValuesList(vals, starelid, inclHLL) print(pstring.format(vals[0], vals[2], ',\n'.join(rowVals))) @@ -153,7 +159,7 @@ def main(): 'options': pgoptions } num_segments = 0 - with closing(pgdb.connect(**connectionInfo)) as connection: + with closing(psycopg2.connect(**connectionInfo)) as connection: with closing(connection.cursor()) as cursor: num_segments = get_num_segments(cursor) sys.stdout.writelines(['\n-- Greenplum database Statistics Dump', @@ -188,7 +194,7 @@ def main(): sys.stdout.flush() try: - with closing(pgdb.connect(**connectionInfo)) as connection: + with closing(psycopg2.connect(**connectionInfo)) as connection: with closing(connection.cursor()) as cursor: dumpTupleCount(cursor) dumpStats(cursor, inclHLL) @@ -197,11 +203,11 @@ def main(): 'which requires some data elements to be included in the output file.\n', 'Please review output file to ensure it is within corporate policy to transport the output file.\n']) - except pgdb.DatabaseError as err: # catch *all* exceptions + except psycopg2.DatabaseError as err: # catch *all* exceptions sys.stderr.write('Error while dumping statistics:\n') sys.stderr.write(err.message) sys.exit(1) if __name__ == "__main__": - main() + main() diff --git a/gpMgmt/bin/minirepro b/gpMgmt/bin/minirepro index 684be762d5c9..682a659ca9ea 100755 --- a/gpMgmt/bin/minirepro +++ b/gpMgmt/bin/minirepro @@ -61,10 +61,10 @@ minirepro gptest -h locahost -U gpadmin -p 4444 -q ~/in.sql -f ~/out.sql import pwd import os, sys, re, json, platform, subprocess -import pgdb +import psycopg2 from optparse import OptionParser from datetime import datetime -from gppylib.utils import formatInsertValuesList, Escape +from gppylib.utils import formatInsertValuesList, escape_string version = '1.13' PATH_PREFIX = '/tmp/' @@ -97,7 +97,7 @@ def get_server_version(cursor): query = "select version()" try: cursor.execute(query) - except pgdb.DatabaseError as e: + except psycopg2.DatabaseError as e: sys.stderr.write('\nError while trying to find GPDB version.\n\n' + str(e) + '\n\n') sys.exit(1) vals = cursor.fetchone() @@ -107,7 +107,7 @@ def get_num_segments(cursor): query = "select count(*) from gp_segment_configuration where role='p' and content >=0;" try: cursor.execute(query) - except pgdb.DatabaseError as e: + except psycopg2.DatabaseError as e: sys.stderr.write('\nError while trying to retrieve number of segments.\n\n' + str(e) + '\n\n') sys.exit(1) vals = cursor.fetchone() @@ -136,7 +136,7 @@ def dump_query(connectionInfo, query_file): with open(query_file, 'r') as query_f: sql_text = query_f.read() - query = "select pg_catalog.gp_dump_query_oids('%s')" % Escape(sql_text) + query = "select pg_catalog.gp_dump_query_oids('%s')" % escape_string(sql_text) toolkit_sql = PATH_PREFIX + 'toolkit.sql' with open(toolkit_sql, 'w') as toolkit_f: @@ -187,7 +187,7 @@ def pg_dump_object(mr_query, connectionInfo, envOpts): out_file = PATH_PREFIX + PGDUMP_FILE dmp_cmd = 'pg_dump -h %s -p %s -U %s -sxO %s' % connectionInfo dmp_cmd = "%s --relation-oids %s --function-oids %s -f %s" % \ - (dmp_cmd, mr_query.relids, mr_query.funcids, Escape(out_file)) + (dmp_cmd, mr_query.relids, mr_query.funcids, escape_string(out_file)) print(dmp_cmd) p = subprocess.Popen(dmp_cmd, shell=True, stderr=subprocess.PIPE, env=envOpts) _, errormsg = p.communicate() @@ -213,11 +213,17 @@ def dump_tuple_count(cur, oid_str, f_out): for col, val, typ in zip(columns[2:], vals[2:], types): # i.e. relpages = 1::int, reltuples = 1.0::real lines.append('\t%s = %s::%s' % (col, val, typ)) - updateStmt = templateStmt.format(Escape(',\n'.join(lines)), Escape(vals[0]), Escape(vals[1])) + updateStmt = templateStmt.format(escape_string(',\n'.join(lines)), escape_string(vals[0]), escape_string(vals[1])) f_out.writelines(updateStmt) def dump_stats(cur, oid_str, f_out, inclHLL): - query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, pgs.* ' \ + query = 'SELECT pgc.relname, pgn.nspname, pga.attname, pgtn.nspname, pgt.typname, ' \ + 'pgs.starelid, pgs.staattnum, pgs.stainherit, pgs.stanullfrac, pgs.stawidth, pgs.stadistinct, ' \ + 'pgs.stakind1, pgs.stakind2, pgs.stakind3, pgs.stakind4, pgs.stakind5, ' \ + 'pgs.staop1, pgs.staop2, pgs.staop3, pgs.staop4, pgs.staop5, ' \ + 'pgs.stacoll1, pgs.stacoll2, pgs.stacoll3, pgs.stacoll4, pgs.stacoll5, ' \ + 'pgs.stanumbers1, pgs.stanumbers2, pgs.stanumbers3, pgs.stanumbers4, pgs.stanumbers5, ' \ + 'pgs.stavalues1::text::text[], pgs.stavalues2::text::text[], pgs.stavalues3::text::text[], pgs.stavalues4::text::text[], pgs.stavalues5::text::text[] ' \ 'FROM pg_class pgc, pg_statistic pgs, pg_namespace pgn, pg_attribute pga, pg_type pgt, pg_namespace pgtn ' \ 'WHERE pgc.relnamespace = pgn.oid and pgc.oid in (%s) ' \ 'and pgn.nspname NOT LIKE \'pg_temp_%%\' ' \ @@ -239,7 +245,7 @@ def dump_stats(cur, oid_str, f_out, inclHLL): for vals in result_iter(cur): schemaname = vals[1] - starelid = "'%s.%s'::regclass" % (Escape(vals[1]), Escape(vals[0])) + starelid = "'%s.%s'::regclass" % (escape_string(vals[1]), escape_string(vals[0])) rowVals = formatInsertValuesList(vals, starelid, inclHLL) # For non-catalog tables we don't need to delete stats first @@ -248,7 +254,7 @@ def dump_stats(cur, oid_str, f_out, inclHLL): if schemaname != 'pg_catalog': linecomment = '-- ' # This will comment out the DELETE query - f_out.writelines(pstring.format(Escape(vals[0]), Escape(vals[2]), linecomment, starelid, vals[6], ',\n'.join(rowVals))) + f_out.writelines(pstring.format(escape_string(vals[0]), escape_string(vals[2]), linecomment, starelid, vals[6], ',\n'.join(rowVals))) def main(): parser = parse_cmd_line() @@ -299,7 +305,7 @@ def main(): } print("Connecting to database: host=%s, port=%s, user=%s, db=%s ..." % connectionInfo) - conn = pgdb.connect(**connectionDict) + conn = psycopg2.connect(**connectionDict) cursor = conn.cursor() # get server version, which is dumped to minirepro output file @@ -346,7 +352,7 @@ def main(): # first create schema DDLs print("Writing schema DDLs ...") - table_schemas = ["CREATE SCHEMA %s;\n" % Escape(schema) for schema in mr_query.schemas if schema != 'public'] + table_schemas = ["CREATE SCHEMA %s;\n" % escape_string(schema) for schema in mr_query.schemas if schema != 'public'] f_out.writelines(table_schemas) # write relation and function DDLs @@ -392,4 +398,4 @@ def main(): print('Please review output file to ensure it is within corporate policy to transport the output file.') if __name__ == "__main__": - main() + main() diff --git a/gpMgmt/doc/gpfdist_help b/gpMgmt/doc/gpfdist_help index 52b253675981..93d0abe21837 100755 --- a/gpMgmt/doc/gpfdist_help +++ b/gpMgmt/doc/gpfdist_help @@ -9,6 +9,7 @@ SYNOPSIS gpfdist [-d ] [-p ] [-l ] [-t ] [-S] [-w